diff --git a/app/services/event_study_service.py b/app/services/event_study_service.py
index d107fb9..adac51f 100644
--- a/app/services/event_study_service.py
+++ b/app/services/event_study_service.py
@@ -28,6 +28,10 @@ DRAWDOWN_LOOKBACK = 252
HORIZON_DAYS = 20
WARN_PERCENTILE = 80.0
TRAIN_FRACTION = 0.70
+# Below this many holdout corrections, recall is one event away from a very
+# different headline and should not be read as a property of the score.
+MIN_EVENTS_FOR_CONFIDENCE = 8
+SENSOR_MISMATCH_TOLERANCE = 0.10
def _median(values: list[float]) -> float | None:
@@ -150,18 +154,23 @@ def _warning_series(
dates: list[date],
config: dict,
oas_series: rms.Series | None = None,
-) -> dict[date, float]:
- """Warning score per session, from the monitor's own sensor definitions.
+) -> tuple[dict[date, float], dict[date, int]]:
+ """Warning score per session plus how many sensors backed it.
v2 re-derived this by hand from ``WARNING_WEIGHTS`` and so would have kept
measuring the old construct after a scoring change. Since v3 dropped
fundamentals from the score, this is now exactly the live Warning score
rather than a technical-only approximation of it.
+
+ The sensor count matters because the score renormalises over whatever is
+ available: a session backed by two sensors is not drawn from the same
+ distribution as one backed by three, and the frozen threshold assumes it is.
"""
tickers = config["tickers"]
smh_full = prices.get(tickers["leaders"][0], [])
spy_full = prices.get(tickers["market"], [])
out: dict[date, float] = {}
+ backing: dict[date, int] = {}
for session in dates:
sensors = rms.warning_sensor_scores(
breadth_divergence.get(session),
@@ -172,7 +181,44 @@ def _warning_series(
score = rms.score_warning_sensors(sensors)
if score is not None:
out[session] = round(score, 2)
- return out
+ backing[session] = sum(1 for value in sensors.values() if value is not None)
+ return out, backing
+
+
+def _reliability(
+ dates: list[date],
+ split: int,
+ backing: dict[date, int],
+ events_detected: int,
+ events_in_holdout: int,
+) -> dict:
+ """How far the headline metrics can actually be trusted.
+
+ Two things repeatedly invite over-reading this report:
+
+ * The holdout carries only the corrections that fall in the last 30% of the
+ sample. A "2/4" is one event away from "3/4", and in practice the events
+ that flip are decided by where the frozen threshold happens to land rather
+ than by whether the score saw anything.
+ * The score renormalises over available sensors, so a training window that
+ predates a sensor's history freezes a threshold on a different construct
+ than the holdout is measured against.
+ """
+ expected = len(rms.WARNING_WEIGHTS)
+ train = [backing[d] for d in dates[:split] if d in backing]
+ holdout = [backing[d] for d in dates[split:] if d in backing]
+ train_full = sum(1 for n in train if n == expected) / len(train) if train else 0.0
+ holdout_full = sum(1 for n in holdout if n == expected) / len(holdout) if holdout else 0.0
+ return {
+ "events_detected": events_detected,
+ "events_in_holdout": events_in_holdout,
+ "minimum_events": MIN_EVENTS_FOR_CONFIDENCE,
+ "underpowered": events_in_holdout < MIN_EVENTS_FOR_CONFIDENCE,
+ "sensors_expected": expected,
+ "train_full_sensor_share": round(train_full * 100, 1),
+ "holdout_full_sensor_share": round(holdout_full * 100, 1),
+ "sensor_coverage_mismatch": abs(train_full - holdout_full) > SENSOR_MISMATCH_TOLERANCE,
+ }
async def run_event_study(
@@ -196,7 +242,7 @@ async def run_event_study(
)
divergence = breadth_service.compute_divergence_series(breadth, benchmark)
oas_series = await rms._fetch_fred_series("BAMLH0A0HYM2", start, end)
- warning = _warning_series(prices, divergence, dates, config, oas_series)
+ warning, backing = _warning_series(prices, divergence, dates, config, oas_series)
# The credit sensor cannot reach back as far as the price history does (the
# upstream series is capped at ~3 years), so the earlier part of the sample
# scores on W1+W2 alone via renormalisation. Report where W3 starts rather
@@ -218,6 +264,8 @@ async def run_event_study(
metrics["false_alarms"] / (holdout_sessions / 252.0), 2
)
+ reliability = _reliability(dates, split, backing, len(all_events), len(holdout_events))
+
basket_asof = date.fromisoformat(config["basket_asof"])
retrospective = dates[split] < basket_asof
evaluation = "exploratory" if retrospective else "holdout"
@@ -230,7 +278,14 @@ async def run_event_study(
f"{evaluation.capitalize()} chronological test: warning episodes preceded "
f"{metrics['events_warned']}/{metrics['events']} 10% corrections; "
f"{metrics['events_missed']} missed, {metrics['false_alarms_per_year']:.1f} "
- f"false alarms/year, {lead_text}."
+ f"false alarms/year, {lead_text}. "
+ f"{metrics['events']} of {reliability['events_detected']} detected corrections "
+ f"fall in the test period"
+ + (
+ f"; too few to read recall as a property of the score."
+ if reliability["underpowered"]
+ else "."
+ )
)
per_event = metrics.pop("per_event")
@@ -262,6 +317,7 @@ async def run_event_study(
"holdout_sessions": holdout_sessions,
},
"metrics": metrics,
+ "reliability": reliability,
"events": per_event,
"recent_breadth": [
{"date": d.isoformat(), "breadth": breadth[d], "warning": warning.get(d)}
@@ -273,8 +329,11 @@ async def run_event_study(
"event": "regime_event_study_complete",
"evaluation": evaluation,
"events": metrics["events"],
+ "events_detected": reliability["events_detected"],
"warned": metrics["events_warned"],
"false_alarms_per_year": metrics["false_alarms_per_year"],
+ "underpowered": reliability["underpowered"],
+ "sensor_coverage_mismatch": reliability["sensor_coverage_mismatch"],
}))
return report
diff --git a/docs/research/regime-monitor-v3.md b/docs/research/regime-monitor-v3.md
index 92254ec..f1d9443 100644
--- a/docs/research/regime-monitor-v3.md
+++ b/docs/research/regime-monitor-v3.md
@@ -160,6 +160,40 @@ A cached report is discarded when its methodology no longer matches, so the pane
reverts to "not run yet" after a bump rather than showing stale numbers. **Re-run
the Event Study job after cutting over to v3.**
+### Reading the result
+
+The report carries a `reliability` block and the UI renders its warnings, because
+the headline numbers invite over-reading in two specific ways.
+
+**The holdout is thin.** The study detects 11 corrections across 5 years but the
+70/30 split leaves only 4 in the test period. Recall is therefore one event away
+from a materially different headline, and in practice the event that flips is
+decided by where the frozen threshold happens to land rather than by whether the
+score saw anything. The v3 cutover run illustrates it: v3 scored 2/4 against v2's
+3/4, but "v3 without the credit sensor" scores 3/4 at a *higher* threshold
+(35.5) than shipped v3 misses it at (32.3) — because the alarm rule needs a
+rising edge, and a lower threshold can mean the alarm already fired outside the
+20-session horizon and never reset below. Below `MIN_EVENTS_FOR_CONFIDENCE`
+holdout events the report says so explicitly.
+
+Some events carry no information at all for comparison: in that run every
+variant caught 2026-03-06, every variant missed 2026-06-05, and every variant
+"caught" 2025-11-20 with a 1-session lead, which is coincident rather than a
+warning.
+
+**Sensor coverage can straddle the split.** The score renormalises over available
+sensors, so a training window predating a sensor's history freezes the threshold
+on a different construct than the holdout is measured against. At the v3 cutover
+only 39% of training sessions had all three Warning sensors versus 100% of the
+test period, because credit history begins 2023-07-25.
+
+Restricting the threshold to sensor-matched training sessions was tried and is
+*not* the fix: those sessions are a calm recent stretch, so the threshold drops
+from 32.3 to 22.5 and false alarms rise from 3.3 to 8.6 per year. It trades a
+coverage bias for a regime-selection bias. The honest position is that the
+threshold is hypersensitive to window choice at this sample size; the report
+states its limits rather than pretending to a precision it does not have.
+
## Operator rule
Quadrant alerts default off for new/reset configurations. When enabled they
diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts
index 3d924be..e008cfd 100644
--- a/frontend/src/lib/types.ts
+++ b/frontend/src/lib/types.ts
@@ -604,6 +604,18 @@ export interface EventStudyReport {
warn_threshold: number;
basket_hash: string;
basket_asof: string;
+ credit_sensor_from?: string | null;
+ };
+ /** How far the headline metrics can be trusted. See _reliability(). */
+ reliability?: {
+ events_detected: number;
+ events_in_holdout: number;
+ minimum_events: number;
+ underpowered: boolean;
+ sensors_expected: number;
+ train_full_sensor_share: number;
+ holdout_full_sensor_share: number;
+ sensor_coverage_mismatch: boolean;
};
sample?: {
start: string;
diff --git a/frontend/src/pages/RegimePage.tsx b/frontend/src/pages/RegimePage.tsx
index cf8d6df..42af9c5 100644
--- a/frontend/src/pages/RegimePage.tsx
+++ b/frontend/src/pages/RegimePage.tsx
@@ -271,6 +271,32 @@ function EventStudyBody({ report }: { report: EventStudyReport }) {
)}
+ {report.reliability && (report.reliability.underpowered || report.reliability.sensor_coverage_mismatch) && (
+
+ Underpowered. Only {report.reliability.events_in_holdout} of{' '}
+ {report.reliability.events_detected} detected corrections fall in the test period (
+ {report.reliability.minimum_events}+ needed). Recall is one event away from a materially
+ different headline, and which events flip is usually decided by where the frozen threshold
+ lands rather than by what the score saw. Read the direction, not the ratio.
+
+ Sensor coverage differs across the split.{' '}
+ {report.reliability.train_full_sensor_share}% of training sessions had all{' '}
+ {report.reliability.sensors_expected} Warning sensors versus{' '}
+ {report.reliability.holdout_full_sensor_share}% of test sessions
+ {report.params?.credit_sensor_from && ` — credit history begins ${report.params.credit_sensor_from}`}
+ . The score renormalises over what is available, so the threshold was frozen on a partly
+ different construct than it is measured against.
+
The threshold is frozen on the training period and measured on the chronological test period. Reconstructed pre-freeze basket history remains exploratory. diff --git a/tests/unit/test_event_study.py b/tests/unit/test_event_study.py index 72404f1..1db857d 100644 --- a/tests/unit/test_event_study.py +++ b/tests/unit/test_event_study.py @@ -1,4 +1,4 @@ -"""Tests for v2 correction events and warning alarm episodes.""" +"""Tests for v3 correction events, warning alarm episodes, and report caveats.""" from __future__ import annotations @@ -6,7 +6,9 @@ from datetime import date, timedelta from app.services.breadth_service import _breadth_from_closes, compute_divergence_series from app.services.event_study_service import ( + MIN_EVENTS_FOR_CONFIDENCE, _percentile, + _reliability, alarm_episodes, detect_events, evaluate_alarms, @@ -23,6 +25,40 @@ def test_detect_events_uses_rising_edge_and_cooldown(): assert [event["index"] for event in events] == [300, 355] +def test_reliability_flags_a_thin_holdout(): + """2/4 must not read like a property of the score.""" + dates = _days(100) + backing = dict.fromkeys(dates, 3) + + thin = _reliability(dates, 70, backing, events_detected=11, events_in_holdout=4) + assert thin["underpowered"] is True + assert thin["events_detected"] == 11 + assert thin["events_in_holdout"] == 4 + assert thin["minimum_events"] == MIN_EVENTS_FOR_CONFIDENCE + + ample = _reliability(dates, 70, backing, events_detected=20, events_in_holdout=12) + assert ample["underpowered"] is False + + +def test_reliability_flags_a_sensor_coverage_split(): + """The threshold must not be frozen on a different construct than it is tested on. + + Credit history starts partway through the training window, so the score + renormalises over two sensors early and three later. + """ + dates = _days(100) + matched = dict.fromkeys(dates, 3) + assert _reliability(dates, 70, matched, 20, 12)["sensor_coverage_mismatch"] is False + + # Training is 40% three-sensor; the holdout is entirely three-sensor. + split_backing = {d: (3 if index >= 42 else 2) for index, d in enumerate(dates)} + mismatched = _reliability(dates, 70, split_backing, 20, 12) + assert mismatched["sensor_coverage_mismatch"] is True + assert mismatched["train_full_sensor_share"] == 40.0 + assert mismatched["holdout_full_sensor_share"] == 100.0 + assert mismatched["sensors_expected"] == 3 + + def test_percentile_is_fixed_from_supplied_values(): values = [float(value) for value in range(0, 101, 10)] assert _percentile(values, 50) == 50.0