From 83c0555e524013cb0ec4af7dd4b8587fd2a7750d Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Sun, 26 Jul 2026 15:12:53 +0200 Subject: [PATCH] Event study: report its own statistical limits The v3 cutover run scored 2/4 corrections warned against v2's 3/4, which reads like a regression and is not one. Only 4 of the 11 detected corrections fall in the holdout, so recall is one event from a different headline -- and the event that flips is decided by threshold placement, not by what the score saw. "v3 without the credit sensor" catches 2025-02-21 at a *higher* threshold (35.5) than shipped v3 misses it at (32.3), because the alarm rule needs a rising edge and a lower threshold can fire outside the horizon then never reset below. Two caveats are now computed and surfaced rather than left for the reader to infer: - Holdout event count against MIN_EVENTS_FOR_CONFIDENCE. The summary sentence states how many of the detected corrections actually fall in the test period. - Warning-sensor coverage across the split. The score renormalises over what is available, so a training window predating a sensor's history freezes the threshold on a different construct than the holdout is measured against. At the cutover that is 39% of training sessions with all three sensors versus 100% of the test period, credit history beginning 2023-07-25. Restricting the threshold to sensor-matched training sessions was tested and rejected: those sessions are a calm recent stretch, so the threshold falls from 32.3 to 22.5 and false alarms rise from 3.3 to 8.6/yr. It swaps a coverage bias for a regime-selection bias. The report states its limits instead. _warning_series now returns per-session sensor counts alongside the scores. Co-Authored-By: Claude Opus 5 --- app/services/event_study_service.py | 69 ++++++++++++++++++++++++++--- docs/research/regime-monitor-v3.md | 34 ++++++++++++++ frontend/src/lib/types.ts | 12 +++++ frontend/src/pages/RegimePage.tsx | 26 +++++++++++ tests/unit/test_event_study.py | 38 +++++++++++++++- 5 files changed, 173 insertions(+), 6 deletions(-) diff --git a/app/services/event_study_service.py b/app/services/event_study_service.py index d107fb9..adac51f 100644 --- a/app/services/event_study_service.py +++ b/app/services/event_study_service.py @@ -28,6 +28,10 @@ DRAWDOWN_LOOKBACK = 252 HORIZON_DAYS = 20 WARN_PERCENTILE = 80.0 TRAIN_FRACTION = 0.70 +# Below this many holdout corrections, recall is one event away from a very +# different headline and should not be read as a property of the score. +MIN_EVENTS_FOR_CONFIDENCE = 8 +SENSOR_MISMATCH_TOLERANCE = 0.10 def _median(values: list[float]) -> float | None: @@ -150,18 +154,23 @@ def _warning_series( dates: list[date], config: dict, oas_series: rms.Series | None = None, -) -> dict[date, float]: - """Warning score per session, from the monitor's own sensor definitions. +) -> tuple[dict[date, float], dict[date, int]]: + """Warning score per session plus how many sensors backed it. v2 re-derived this by hand from ``WARNING_WEIGHTS`` and so would have kept measuring the old construct after a scoring change. Since v3 dropped fundamentals from the score, this is now exactly the live Warning score rather than a technical-only approximation of it. + + The sensor count matters because the score renormalises over whatever is + available: a session backed by two sensors is not drawn from the same + distribution as one backed by three, and the frozen threshold assumes it is. """ tickers = config["tickers"] smh_full = prices.get(tickers["leaders"][0], []) spy_full = prices.get(tickers["market"], []) out: dict[date, float] = {} + backing: dict[date, int] = {} for session in dates: sensors = rms.warning_sensor_scores( breadth_divergence.get(session), @@ -172,7 +181,44 @@ def _warning_series( score = rms.score_warning_sensors(sensors) if score is not None: out[session] = round(score, 2) - return out + backing[session] = sum(1 for value in sensors.values() if value is not None) + return out, backing + + +def _reliability( + dates: list[date], + split: int, + backing: dict[date, int], + events_detected: int, + events_in_holdout: int, +) -> dict: + """How far the headline metrics can actually be trusted. + + Two things repeatedly invite over-reading this report: + + * The holdout carries only the corrections that fall in the last 30% of the + sample. A "2/4" is one event away from "3/4", and in practice the events + that flip are decided by where the frozen threshold happens to land rather + than by whether the score saw anything. + * The score renormalises over available sensors, so a training window that + predates a sensor's history freezes a threshold on a different construct + than the holdout is measured against. + """ + expected = len(rms.WARNING_WEIGHTS) + train = [backing[d] for d in dates[:split] if d in backing] + holdout = [backing[d] for d in dates[split:] if d in backing] + train_full = sum(1 for n in train if n == expected) / len(train) if train else 0.0 + holdout_full = sum(1 for n in holdout if n == expected) / len(holdout) if holdout else 0.0 + return { + "events_detected": events_detected, + "events_in_holdout": events_in_holdout, + "minimum_events": MIN_EVENTS_FOR_CONFIDENCE, + "underpowered": events_in_holdout < MIN_EVENTS_FOR_CONFIDENCE, + "sensors_expected": expected, + "train_full_sensor_share": round(train_full * 100, 1), + "holdout_full_sensor_share": round(holdout_full * 100, 1), + "sensor_coverage_mismatch": abs(train_full - holdout_full) > SENSOR_MISMATCH_TOLERANCE, + } async def run_event_study( @@ -196,7 +242,7 @@ async def run_event_study( ) divergence = breadth_service.compute_divergence_series(breadth, benchmark) oas_series = await rms._fetch_fred_series("BAMLH0A0HYM2", start, end) - warning = _warning_series(prices, divergence, dates, config, oas_series) + warning, backing = _warning_series(prices, divergence, dates, config, oas_series) # The credit sensor cannot reach back as far as the price history does (the # upstream series is capped at ~3 years), so the earlier part of the sample # scores on W1+W2 alone via renormalisation. Report where W3 starts rather @@ -218,6 +264,8 @@ async def run_event_study( metrics["false_alarms"] / (holdout_sessions / 252.0), 2 ) + reliability = _reliability(dates, split, backing, len(all_events), len(holdout_events)) + basket_asof = date.fromisoformat(config["basket_asof"]) retrospective = dates[split] < basket_asof evaluation = "exploratory" if retrospective else "holdout" @@ -230,7 +278,14 @@ async def run_event_study( f"{evaluation.capitalize()} chronological test: warning episodes preceded " f"{metrics['events_warned']}/{metrics['events']} 10% corrections; " f"{metrics['events_missed']} missed, {metrics['false_alarms_per_year']:.1f} " - f"false alarms/year, {lead_text}." + f"false alarms/year, {lead_text}. " + f"{metrics['events']} of {reliability['events_detected']} detected corrections " + f"fall in the test period" + + ( + f"; too few to read recall as a property of the score." + if reliability["underpowered"] + else "." + ) ) per_event = metrics.pop("per_event") @@ -262,6 +317,7 @@ async def run_event_study( "holdout_sessions": holdout_sessions, }, "metrics": metrics, + "reliability": reliability, "events": per_event, "recent_breadth": [ {"date": d.isoformat(), "breadth": breadth[d], "warning": warning.get(d)} @@ -273,8 +329,11 @@ async def run_event_study( "event": "regime_event_study_complete", "evaluation": evaluation, "events": metrics["events"], + "events_detected": reliability["events_detected"], "warned": metrics["events_warned"], "false_alarms_per_year": metrics["false_alarms_per_year"], + "underpowered": reliability["underpowered"], + "sensor_coverage_mismatch": reliability["sensor_coverage_mismatch"], })) return report diff --git a/docs/research/regime-monitor-v3.md b/docs/research/regime-monitor-v3.md index 92254ec..f1d9443 100644 --- a/docs/research/regime-monitor-v3.md +++ b/docs/research/regime-monitor-v3.md @@ -160,6 +160,40 @@ A cached report is discarded when its methodology no longer matches, so the pane reverts to "not run yet" after a bump rather than showing stale numbers. **Re-run the Event Study job after cutting over to v3.** +### Reading the result + +The report carries a `reliability` block and the UI renders its warnings, because +the headline numbers invite over-reading in two specific ways. + +**The holdout is thin.** The study detects 11 corrections across 5 years but the +70/30 split leaves only 4 in the test period. Recall is therefore one event away +from a materially different headline, and in practice the event that flips is +decided by where the frozen threshold happens to land rather than by whether the +score saw anything. The v3 cutover run illustrates it: v3 scored 2/4 against v2's +3/4, but "v3 without the credit sensor" scores 3/4 at a *higher* threshold +(35.5) than shipped v3 misses it at (32.3) — because the alarm rule needs a +rising edge, and a lower threshold can mean the alarm already fired outside the +20-session horizon and never reset below. Below `MIN_EVENTS_FOR_CONFIDENCE` +holdout events the report says so explicitly. + +Some events carry no information at all for comparison: in that run every +variant caught 2026-03-06, every variant missed 2026-06-05, and every variant +"caught" 2025-11-20 with a 1-session lead, which is coincident rather than a +warning. + +**Sensor coverage can straddle the split.** The score renormalises over available +sensors, so a training window predating a sensor's history freezes the threshold +on a different construct than the holdout is measured against. At the v3 cutover +only 39% of training sessions had all three Warning sensors versus 100% of the +test period, because credit history begins 2023-07-25. + +Restricting the threshold to sensor-matched training sessions was tried and is +*not* the fix: those sessions are a calm recent stretch, so the threshold drops +from 32.3 to 22.5 and false alarms rise from 3.3 to 8.6 per year. It trades a +coverage bias for a regime-selection bias. The honest position is that the +threshold is hypersensitive to window choice at this sample size; the report +states its limits rather than pretending to a precision it does not have. + ## Operator rule Quadrant alerts default off for new/reset configurations. When enabled they diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 3d924be..e008cfd 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -604,6 +604,18 @@ export interface EventStudyReport { warn_threshold: number; basket_hash: string; basket_asof: string; + credit_sensor_from?: string | null; + }; + /** How far the headline metrics can be trusted. See _reliability(). */ + reliability?: { + events_detected: number; + events_in_holdout: number; + minimum_events: number; + underpowered: boolean; + sensors_expected: number; + train_full_sensor_share: number; + holdout_full_sensor_share: number; + sensor_coverage_mismatch: boolean; }; sample?: { start: string; diff --git a/frontend/src/pages/RegimePage.tsx b/frontend/src/pages/RegimePage.tsx index cf8d6df..42af9c5 100644 --- a/frontend/src/pages/RegimePage.tsx +++ b/frontend/src/pages/RegimePage.tsx @@ -271,6 +271,32 @@ function EventStudyBody({ report }: { report: EventStudyReport }) { )} + {report.reliability && (report.reliability.underpowered || report.reliability.sensor_coverage_mismatch) && ( + +
+ {report.reliability.underpowered && ( +

+ Underpowered. Only {report.reliability.events_in_holdout} of{' '} + {report.reliability.events_detected} detected corrections fall in the test period ( + {report.reliability.minimum_events}+ needed). Recall is one event away from a materially + different headline, and which events flip is usually decided by where the frozen threshold + lands rather than by what the score saw. Read the direction, not the ratio. +

+ )} + {report.reliability.sensor_coverage_mismatch && ( +

+ Sensor coverage differs across the split.{' '} + {report.reliability.train_full_sensor_share}% of training sessions had all{' '} + {report.reliability.sensors_expected} Warning sensors versus{' '} + {report.reliability.holdout_full_sensor_share}% of test sessions + {report.params?.credit_sensor_from && ` — credit history begins ${report.params.credit_sensor_from}`} + . The score renormalises over what is available, so the threshold was frozen on a partly + different construct than it is measured against. +

+ )} +
+ + )}

The threshold is frozen on the training period and measured on the chronological test period. Reconstructed pre-freeze basket history remains exploratory. diff --git a/tests/unit/test_event_study.py b/tests/unit/test_event_study.py index 72404f1..1db857d 100644 --- a/tests/unit/test_event_study.py +++ b/tests/unit/test_event_study.py @@ -1,4 +1,4 @@ -"""Tests for v2 correction events and warning alarm episodes.""" +"""Tests for v3 correction events, warning alarm episodes, and report caveats.""" from __future__ import annotations @@ -6,7 +6,9 @@ from datetime import date, timedelta from app.services.breadth_service import _breadth_from_closes, compute_divergence_series from app.services.event_study_service import ( + MIN_EVENTS_FOR_CONFIDENCE, _percentile, + _reliability, alarm_episodes, detect_events, evaluate_alarms, @@ -23,6 +25,40 @@ def test_detect_events_uses_rising_edge_and_cooldown(): assert [event["index"] for event in events] == [300, 355] +def test_reliability_flags_a_thin_holdout(): + """2/4 must not read like a property of the score.""" + dates = _days(100) + backing = dict.fromkeys(dates, 3) + + thin = _reliability(dates, 70, backing, events_detected=11, events_in_holdout=4) + assert thin["underpowered"] is True + assert thin["events_detected"] == 11 + assert thin["events_in_holdout"] == 4 + assert thin["minimum_events"] == MIN_EVENTS_FOR_CONFIDENCE + + ample = _reliability(dates, 70, backing, events_detected=20, events_in_holdout=12) + assert ample["underpowered"] is False + + +def test_reliability_flags_a_sensor_coverage_split(): + """The threshold must not be frozen on a different construct than it is tested on. + + Credit history starts partway through the training window, so the score + renormalises over two sensors early and three later. + """ + dates = _days(100) + matched = dict.fromkeys(dates, 3) + assert _reliability(dates, 70, matched, 20, 12)["sensor_coverage_mismatch"] is False + + # Training is 40% three-sensor; the holdout is entirely three-sensor. + split_backing = {d: (3 if index >= 42 else 2) for index, d in enumerate(dates)} + mismatched = _reliability(dates, 70, split_backing, 20, 12) + assert mismatched["sensor_coverage_mismatch"] is True + assert mismatched["train_full_sensor_share"] == 40.0 + assert mismatched["holdout_full_sensor_share"] == 100.0 + assert mismatched["sensors_expected"] == 3 + + def test_percentile_is_fixed_from_supplied_values(): values = [float(value) for value in range(0, 101, 10)] assert _percentile(values, 50) == 50.0