"""Chronological validation for the AI/Tech Risk Monitor warning score. The outcome is a 10% correction in the leader, never a regime break. Two rules are measured against it, and they answer different questions: * **shipped** -- the quadrant-change rule that actually reaches Telegram (``alert_service._collect_regime_quadrant``). Its thresholds are fixed constants chosen by scenario arithmetic, so nothing is fitted, so there is no training set to protect and the whole sample is evaluable. This is the headline. * **fitted** -- the original study: an 80th-percentile Warning threshold frozen on the first 70% of sessions and measured on the last 30%. Kept because it is what the methodology document reports, and because a fitted threshold is a genuinely different question -- but it is measured on the four corrections that happen to fall in the holdout, which is too few to read as a property of the score. Both are scored by the same ``evaluate_alarms`` harness, alongside ablations (does the quadrant machinery earn its place?), external baselines (does the score earn its complexity?), and a random-alarm null (is any of this better than chance?). Without those rows a bare "2 of 4" is unreadable in either direction. The fundamental channel is compared, never fused. It appears as its own rule (transitions into an adverse state), as a confluence gate (a market crossing kept only when the state agrees), and as a market-only comparator over the identical window -- because with ~10 correction events and almost no fundamental history, any weight that combined it with the market axes would be a policy preference presented as a measurement. Those three rows are **coverage-matched**: scored only on the sessions where the channel had usable context and on the corrections whose warning horizon fell inside it, and marked ``measurable: false`` until enough corrections are covered. A fundamental rule scores zero whether it is wrong or merely absent, so scoring it over the market rows' full sample would turn a fortnight of observations into a 0/10 that reads as a failed test. Still labelled exploratory while the fixed breadth basket is reconstructed before its freeze date. """ from __future__ import annotations import json import logging import random from datetime import date, datetime, timedelta, timezone from sqlalchemy.ext.asyncio import AsyncSession from app.services import breadth_service, settings_store from app.services import regime_monitor_service as rms from app.services.admin_service import update_setting from app.services.alert_service import ( QUAD_COOLDOWN_DAYS, QUAD_MARGIN, QUAD_X_DIV, QUAD_Y_DIV, _classify_quadrant, ) logger = logging.getLogger(__name__) KEY_REPORT = "regime_event_study" # Report shape, independent of METHODOLOGY. A cached report from an older shape # parses fine and reports the current methodology, so without this check the # panel would render a report missing half its blocks. Bumping discards the cache # the way a methodology change does -- and it is the *only* thing that does so # here, because the fundamental-channel rework left METHODOLOGY on v4 (the scores # did not change), so the methodology check cannot catch a stale report. STUDY_SCHEMA = 3 EVENT_THRESHOLD_PCT = 10.0 EVENT_COOLDOWN_DAYS = 40 DRAWDOWN_LOOKBACK = 252 HORIZON_DAYS = 20 WARN_PERCENTILE = 80.0 TRAIN_FRACTION = 0.70 # Below this many holdout corrections, recall is one event away from a very # different headline and should not be read as a property of the score. MIN_EVENTS_FOR_CONFIDENCE = 8 SENSOR_MISMATCH_TOLERANCE = 0.10 # _collect_regime_quadrant confirms against get_regime_history(db, days=14), so a # prior session older than that window is not available to confirm with. QUAD_HISTORY_DAYS = 14 # Quadrants with Warning above its divider: "1" early warning, "2" active stress. WARNING_QUADRANTS = ("1", "2") STRESS_QUADRANT = ("2",) # Draws for the random-alarm null. Seeded, because a cached report that moves # on re-run for RNG reasons is worse than no report. NULL_DRAWS = 2000 NULL_SEED = 20260812 BASELINE_SMA_WINDOW = 50 BASELINE_VIX_LEVEL = 20.0 def _median(values: list[float]) -> float | None: if not values: return None ordered = sorted(values) middle = len(ordered) // 2 return ( float(ordered[middle]) if len(ordered) % 2 else (ordered[middle - 1] + ordered[middle]) / 2.0 ) def _percentile(values: list[float], pct: float) -> float | None: ordered = sorted(v for v in values if v is not None) if not ordered: return None position = (len(ordered) - 1) * pct / 100.0 lower = int(position) upper = min(lower + 1, len(ordered) - 1) return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) def detect_events( closes: list[float], dates: list[date], threshold_pct: float = EVENT_THRESHOLD_PCT, lookback: int = DRAWDOWN_LOOKBACK, cooldown: int = EVENT_COOLDOWN_DAYS, ) -> list[dict]: """Rising-edge corrections from the trailing 52-week high.""" events: list[dict] = [] previous_drawdown = 0.0 last_event = -10**9 for index, close in enumerate(closes): high = max(closes[max(0, index - lookback + 1): index + 1]) drawdown = (high - close) / high * 100.0 if high > 0 else 0.0 if ( drawdown >= threshold_pct and previous_drawdown < threshold_pct and index - last_event >= cooldown ): events.append({ "date": dates[index].isoformat(), "index": index, "depth_pct": round(drawdown, 1), }) last_event = index previous_drawdown = drawdown return events def alarm_episodes( indicator: dict[date, float], dates: list[date], threshold: float, start_index: int = 1, ) -> list[int]: """Indices where the warning crosses upward; it must reset below first.""" alarms: list[int] = [] was_high = False if start_index > 0: previous = indicator.get(dates[start_index - 1]) was_high = previous is not None and previous >= threshold for index in range(start_index, len(dates)): value = indicator.get(dates[index]) if value is None: continue high = value >= threshold if high and not was_high: alarms.append(index) was_high = high return alarms def evaluate_alarms( alarm_indices: list[int], event_indices: list[int], dates: list[date], horizon: int = HORIZON_DAYS, ) -> dict: """Event recall, episode false alarms, and lead time for one holdout.""" leads: list[float] = [] per_event: list[dict] = [] warned = 0 for event_index in event_indices: matching = [ alarm for alarm in alarm_indices if 0 < event_index - alarm <= horizon ] lead = max((event_index - alarm for alarm in matching), default=None) if lead is not None: warned += 1 leads.append(float(lead)) per_event.append({ "date": dates[event_index].isoformat(), "warned": lead is not None, "lead_days": lead, }) false_alarms = sum( 1 for alarm in alarm_indices if not any(0 < event - alarm <= horizon for event in event_indices) ) return { "events": len(event_indices), "events_warned": warned, "events_missed": len(event_indices) - warned, "alarm_episodes": len(alarm_indices), "false_alarms": false_alarms, "median_lead_days": _median(leads), "per_event": per_event, } def _score_rule( alarm_indices: list[int], event_indices: list[int], dates: list[date], horizon: int, sessions: int, ) -> dict: """``evaluate_alarms`` plus the annualised false-alarm rate for one rule. The rate is ``None`` when the rule had no eligible sessions. Dividing by a tiny floor instead produced 5e9 alarms/year for a coverage-matched rule with an empty window -- a number that means "undefined" while looking like a measurement, which is the failure mode this whole panel is built to avoid. """ metrics = evaluate_alarms(alarm_indices, event_indices, dates, horizon) metrics["false_alarms_per_year"] = ( round(metrics["false_alarms"] / (sessions / 252.0), 2) if sessions > 0 else None ) return metrics # --------------------------------------------------------------------------- # The shipped rule # --------------------------------------------------------------------------- def _axis_rows( prices: dict[str, rms.Series], vix_series: rms.Series | None, oas_series: rms.Series | None, breadth_series: rms.Series | None, divergence_series: rms.Series | None, dates: list[date], config: dict, observations: list[dict] | None = None, ) -> dict[date, dict]: """State and Warning per session, from the function that writes snapshots. Calling ``_compute_index`` rather than re-deriving the two axes is the same anti-drift argument that produced ``warning_sensor_scores``: the v2 study re-derived Warning by hand and would have kept measuring the old construct through a scoring change. State has no such shared helper, so the whole snapshot builder is the shared definition. ``observations`` is the point-in-time fundamental series. It does not enter either score -- the fundamental channel is categorical and read by confluence -- but the per-session ``fundamental_state`` it produces is what the confluence rule below is measured on, so it has to be the same series production reports from. Every variant in this module reads its Warning from these rows, so there is no second derivation to fall out of step. """ rows: dict[date, dict] = {} for session in dates: snapshot = rms._compute_index( prices, vix_series, oas_series, {}, config, session, breadth_series=breadth_series, divergence_series=divergence_series, observations=observations or [], ) state = snapshot["state"] warning = snapshot["warning"] rows[session] = { "state": state.get("score"), "warning": warning.get("score"), "fundamental_state": (snapshot.get("fundamental_context") or {}).get("state"), # `usable`, not `available`: a stale observation keeps its state for # display but stops counting as evidence, and an observation whose # extraction failed on everything is fresh but knows nothing. Either # one counted here would inflate the covered window with sessions the # channel could not have contributed to. "fundamental_usable": bool( (snapshot.get("fundamental_context") or {}).get("usable") ), "state_coverage": state.get("coverage") or 0.0, "warning_coverage": warning.get("coverage") or 0.0, # The score renormalises over available sensors, so a session backed # by two is not drawn from the same distribution as one backed by # three, and a frozen threshold assumes it is. "warning_sensors": len(warning.get("available_pillars") or []), "inputs_fresh": bool((snapshot.get("data_quality") or {}).get("inputs_fresh")), } return rows def _publishable(row: dict | None) -> bool: """What ``get_regime_history`` leaves for the alert to confirm against. Deliberately not freshness-gated: ``_collect_regime_quadrant`` checks ``is_fresh`` on today's live reading only, while the prior session comes from stored history where the only filter is a published band on both axes. """ return ( row is not None and row["state"] is not None and row["warning"] is not None and row["state_coverage"] >= rms.MIN_COVERAGE and row["warning_coverage"] >= rms.MIN_COVERAGE ) def _prior_publishable( rows: dict[date, dict], dates: list[date], index: int, history_days: int ) -> dict | None: """``valid[-2]``: the previous published session inside the 14-day window. The monitor writes today's snapshot before the alert step runs (``job_catalog._DAILY_PIPELINE_STEPS``), so ``valid[-1]`` is today and this is genuinely the prior session rather than t-2. """ cutoff = dates[index] - timedelta(days=history_days) for position in range(index - 1, -1, -1): if dates[position] < cutoff: return None candidate = rows.get(dates[position]) if _publishable(candidate): return candidate return None def replay_quadrant_changes( rows: dict[date, dict], dates: list[date], state_divider: float = QUAD_X_DIV, warning_divider: float = QUAD_Y_DIV, margin: float = QUAD_MARGIN, cooldown_days: int = QUAD_COOLDOWN_DAYS, history_days: int = QUAD_HISTORY_DAYS, ) -> list[dict]: """Every quadrant change the shipped alert would have sent, in order. A faithful replay of ``_collect_regime_quadrant``, including three details a state machine written from first principles gets wrong: * the prior session is classified against the *current baseline*, not against its own predecessor, so confirmation asks "did yesterday already look like this change" rather than "did yesterday change too"; * the baseline advances only when an alert actually fires, so a change that fails confirmation or cooldown is re-evaluated against the old quadrant on the next session rather than being forgotten; * one cooldown is shared by every quadrant change, so a 3->4 alert can swallow a 4->2 alert three days later. Returns the fires themselves rather than alarm indices, because which transitions count as a *warning* is the caller's question: entering Warning-high territory and entering both-high territory are different rules over the same replay. """ fires: list[dict] = [] baseline: str | None = None baseline_date: date | None = None for index, session in enumerate(dates): row = rows.get(session) if not _publishable(row) or not row["inputs_fresh"]: continue x, y = float(row["state"]), float(row["warning"]) if baseline is None: # seeds silently, exactly as a fresh install does baseline = _classify_quadrant(x, y, None, margin, state_divider, warning_divider) baseline_date = session continue new_quadrant = _classify_quadrant(x, y, baseline, margin, state_divider, warning_divider) if new_quadrant == baseline: continue prior = _prior_publishable(rows, dates, index, history_days) if prior is None: continue prior_quadrant = _classify_quadrant( float(prior["state"]), float(prior["warning"]), baseline, margin, state_divider, warning_divider, ) if prior_quadrant != new_quadrant: continue if baseline_date is not None and (session - baseline_date).days < cooldown_days: continue fires.append({ "index": index, "date": session.isoformat(), "from": baseline, "to": new_quadrant, "state": x, "warning": y, }) baseline, baseline_date = new_quadrant, session return fires def entry_alarms(fires: list[dict], entry: tuple[str, ...]) -> list[int]: """Fires that *enter* the given quadrant set from outside it.""" return [f["index"] for f in fires if f["to"] in entry and f["from"] not in entry] # --------------------------------------------------------------------------- # Ablations, baselines, null # --------------------------------------------------------------------------- def _usable_adverse(rows: dict[date, dict], session: date) -> bool: """Adverse *and* still within its staleness horizon. Both callers need this pair, and neither may use the state alone: the state survives going stale so the card can show it, which would otherwise let a months-old read confirm crossings indefinitely. """ row = rows.get(session) or {} return row.get("fundamental_state") == "adverse" and bool(row.get("fundamental_usable")) def adverse_episodes( rows: dict[date, dict], dates: list[date], start_index: int ) -> list[int]: """Sessions where the fundamental state *becomes* usably adverse. The market rules alarm on a rising-edge crossing; a categorical state has no crossing, so its analogue is the transition into ``adverse``. That keeps the row comparable with every other row in the table rather than counting every day the state happens to sit there. """ alarms: list[int] = [] was_adverse = start_index > 0 and _usable_adverse(rows, dates[start_index - 1]) for index in range(start_index, len(dates)): if dates[index] not in rows: continue adverse = _usable_adverse(rows, dates[index]) if adverse and not was_adverse: alarms.append(index) was_adverse = adverse return alarms def confluence_episodes( warning_alarms: list[int], rows: dict[date, dict], dates: list[date] ) -> list[int]: """Warning crossings that happen while the fundamental state is usably adverse. Deliberately gated on the market crossing rather than on either channel moving: it preserves the rising-edge semantics every other row uses, so the column measures "does requiring fundamental agreement help?" instead of a differently-shaped rule that cannot be compared with the others. """ return [index for index in warning_alarms if _usable_adverse(rows, dates[index])] def covered_events( event_indices: list[int], rows: dict[date, dict], dates: list[date], horizon: int, ) -> list[int]: """Corrections a fundamental rule actually had a chance to warn about. An alarm counts only if it fires in ``[event - horizon, event - 1]``, so a correction is *coverable* only if the channel had usable context somewhere in that window. Scoring these rules against every correction instead would make one day of observation render as 0/10 -- an untested rule reported as a failed one, which is the exact mistake the ``measurable`` flag exists to prevent for the empty-table case. """ covered: list[int] = [] for event_index in event_indices: window = range(max(0, event_index - horizon), event_index) if any( bool((rows.get(dates[index]) or {}).get("fundamental_usable")) for index in window ): covered.append(event_index) return covered def eligible_sessions( rows: dict[date, dict], dates: list[date], start_index: int ) -> int: """Sessions a fundamental rule could have fired on, for the FA/year rate. Annualising over the whole window instead would divide a rule's false alarms by years in which it was structurally incapable of firing, reporting a flattering rate that means nothing. """ return sum( 1 for session in dates[start_index:] if bool((rows.get(session) or {}).get("fundamental_usable")) ) def below_average_series( series: rms.Series, window: int = BASELINE_SMA_WINDOW ) -> dict[date, float]: """100 while the close sits under its ``window``-session average, else 0.""" out: dict[date, float] = {} closes = [value for _, value in series] for index, (session, close) in enumerate(series): if index + 1 < window: continue average = sum(closes[index + 1 - window: index + 1]) / window out[session] = 100.0 if close < average else 0.0 return out def _null_model( alarm_count: int, event_indices: list[int], dates: list[date], horizon: int, start_index: int, observed_warned: int, draws: int = NULL_DRAWS, seed: int = NULL_SEED, ) -> dict | None: """Recall from alarms scattered at random over the same evaluable sessions. Drawn only from sessions a real rule could have fired on: over the whole sample the null would be diluted by warm-up sessions and would understate what chance achieves. That matters here -- with ~11 events and a 20-session horizon, a sixth of the sample already sits inside a hit window. Corrections cluster, and uniform placement does not, so this is the floor rather than the bar: an alarm process that clusters would beat it for reasons that have nothing to do with foresight. """ population = range(start_index, len(dates)) if alarm_count <= 0 or not event_indices or alarm_count > len(population): return None rng = random.Random(seed) recalls: list[int] = [] for _ in range(draws): picks = sorted(rng.sample(population, alarm_count)) recalls.append(evaluate_alarms(picks, event_indices, dates, horizon)["events_warned"]) mean = sum(recalls) / len(recalls) variance = sum((value - mean) ** 2 for value in recalls) / len(recalls) return { "draws": draws, "alarms_per_draw": alarm_count, "events": len(event_indices), "mean_warned": round(mean, 2), "sd_warned": round(variance ** 0.5, 2), "observed_warned": observed_warned, "p_at_least_observed": round( sum(1 for value in recalls if value >= observed_warned) / len(recalls), 3 ), } def _reliability( dates: list[date], split: int, backing: dict[date, int], events_detected: int, events_in_holdout: int, ) -> dict: """How far the *fitted* variant's headline metrics can be trusted. Two things repeatedly invite over-reading it: * The holdout carries only the corrections that fall in the last 30% of the sample. A "2/4" is one event away from "3/4", and in practice the events that flip are decided by where the frozen threshold happens to land rather than by whether the score saw anything. * The score renormalises over available sensors, so a training window that predates a sensor's history freezes a threshold on a different construct than the holdout is measured against. Neither applies to the shipped rule, whose thresholds are fixed constants -- but the second one does not vanish, it relocates: see ``_era_split``. """ expected = len(rms.WARNING_WEIGHTS) train = [backing[d] for d in dates[:split] if d in backing] holdout = [backing[d] for d in dates[split:] if d in backing] train_full = sum(1 for n in train if n == expected) / len(train) if train else 0.0 holdout_full = sum(1 for n in holdout if n == expected) / len(holdout) if holdout else 0.0 return { "events_detected": events_detected, "events_in_holdout": events_in_holdout, "minimum_events": MIN_EVENTS_FOR_CONFIDENCE, "underpowered": events_in_holdout < MIN_EVENTS_FOR_CONFIDENCE, "sensors_expected": expected, "train_full_sensor_share": round(train_full * 100, 1), "holdout_full_sensor_share": round(holdout_full * 100, 1), "sensor_coverage_mismatch": abs(train_full - holdout_full) > SENSOR_MISMATCH_TOLERANCE, } def _era_split( alarms: list[int], event_indices: list[int], dates: list[date], horizon: int, start_index: int, credit_from: date | None, ) -> dict | None: """Shipped-rule metrics either side of the credit sensor's first session. Dropping the fitted threshold makes the whole sample evaluable, which is the point -- but most of the extra events sit before 2023-08, where W3 does not exist and Warning renormalises to ``(W1*45 + W2*30)/75``. The fixed 40 divider is then applied to a different construct than it was reasoned about, so the coverage caveat does not disappear with the split; it relocates from the threshold to the score. Reporting the two eras separately is what keeps the fuller sample from being a differently misleading headline. The pre-credit era is close to a "Warning without W3" ablation on real sessions -- and a clean one, because the fundamental channel is not a term in Warning at all, so the two eras differ by W3 and nothing else. That stays true however much fundamental history accumulates. Alarms and events are assigned to eras by index, so an alarm days before the boundary that matched an event days after it lands in the earlier era. With the eras years long and the events sparse, that costs nothing. """ if credit_from is None: return None boundary = next( (index for index, session in enumerate(dates) if session >= credit_from), None ) if boundary is None or boundary <= start_index or boundary >= len(dates): return None def slice_metrics(low: int, high: int) -> dict: sessions = max(0, high - low) metrics = _score_rule( [a for a in alarms if low <= a < high], [e for e in event_indices if low <= e < high], dates, horizon, sessions, ) metrics.pop("per_event", None) metrics["sessions"] = sessions return metrics return { "credit_from": credit_from.isoformat(), "pre_credit": { "label": "W1+W2 only", "start": dates[start_index].isoformat(), "end": dates[boundary - 1].isoformat(), **slice_metrics(start_index, boundary), }, "full_coverage": { "label": "all three sensors", "start": dates[boundary].isoformat(), "end": dates[-1].isoformat(), **slice_metrics(boundary, len(dates)), }, } def _warning_from_rows( rows: dict[date, dict], dates: list[date] ) -> tuple[dict[date, float], dict[date, int]]: """Published Warning per session plus how many sensors backed it. Read off ``_axis_rows`` rather than recomputed. v2 re-derived Warning by hand from ``WARNING_WEIGHTS`` and would have kept measuring the old construct after a scoring change; a second derivation here would have done the same to any later change to how Warning is assembled -- silently, in the fitted variant and the ``warning_bare`` ablation, while the shipped replay moved on without it. """ out: dict[date, float] = {} backing: dict[date, int] = {} for session in dates: row = rows.get(session) if row is None or row["warning"] is None: continue out[session] = float(row["warning"]) backing[session] = int(row["warning_sensors"]) return out, backing def _rule_row( rule_id: str, label: str, kind: str, note: str, alarms: list[int], event_indices: list[int], dates: list[date], horizon: int, sessions: int, measurable: bool = True, ) -> dict: """One comparison row. ``measurable=False`` marks a rule whose *input* is too thin to have been tested, not one that failed. A fundamental rule scores 0/N whether it is wrong or merely absent, and a 0/N sitting in this table would read as tested-and-failed -- the same false precision the whole restructure exists to remove. It stays false until the channel has covered ``MIN_EVENTS_FOR_CONFIDENCE`` corrections, because a 1/1 or 0/2 over a two-week exposure is not a result either. """ metrics = _score_rule(alarms, event_indices, dates, horizon, sessions) metrics.pop("per_event", None) return { "id": rule_id, "label": label, "kind": kind, "note": note, "measurable": measurable, **metrics, } async def run_event_study( db: AsyncSession, threshold_pct: float = EVENT_THRESHOLD_PCT, horizon: int = HORIZON_DAYS, ) -> dict: config = await rms.get_regime_config(db) end = date.today() start = end - timedelta(days=5 * 365 + 30) prices = await rms._fetch_prices(config, start, end) leader = config["tickers"]["leaders"][0] benchmark = sorted(prices.get(leader, []), key=lambda item: item[0]) if len(benchmark) < 500: return {"available": False, "reason": "insufficient benchmark history"} dates = [d for d, _ in benchmark] closes = [value for _, value in benchmark] breadth, _ = await breadth_service.compute_breadth_details( db, config["breadth_basket"], window=200, min_tickers=20 ) divergence = breadth_service.compute_divergence_series(breadth, benchmark) oas_series = await rms._fetch_fred_series("BAMLH0A0HYM2", start, end) # State needs volatility, which the Warning-only study never fetched. vix_series = await rms._fetch_fred_series("VIXCLS", start, end) # The point-in-time fundamental series. It is not in either score; it drives # the categorical channel the confluence rule below is measured on. observations = await rms.get_fundamental_observations(db) # The credit sensor cannot reach back as far as the price history does (the # upstream series is capped at ~3 years), so the earlier part of the sample # scores on W1+W2 alone via renormalisation. Report where W3 starts rather # than letting the threshold quietly straddle two sensor sets. credit_from = oas_series[0][0] if oas_series else None all_events = detect_events(closes, dates, threshold_pct) all_event_indices = [event["index"] for event in all_events] # --- one pass; every rule below reads its Warning from these rows ---- rows = _axis_rows( prices, vix_series, oas_series, rms._mapping_series(breadth), rms._mapping_series(divergence), dates, config, observations, ) warning, backing = _warning_from_rows(rows, dates) fires = replay_quadrant_changes(rows, dates) # Nothing can alarm before the baseline seeds, so every rule is measured from # the same session and the comparison stays like-for-like. seeded = next( ( index for index, session in enumerate(dates) if _publishable(rows.get(session)) and rows[session]["inputs_fresh"] ), None, ) if seeded is None: return {"available": False, "reason": "no session with publishable coverage"} evaluable_start = seeded + 1 evaluable_sessions = max(1, len(dates) - evaluable_start) evaluable_events = [index for index in all_event_indices if index >= evaluable_start] warning_alarms = entry_alarms(fires, WARNING_QUADRANTS) shipped_metrics = _score_rule( warning_alarms, evaluable_events, dates, horizon, evaluable_sessions ) shipped_events = shipped_metrics.pop("per_event") # --- the fitted variant, kept for continuity ------------------------- split = max(1, min(len(dates) - 1, int(len(dates) * TRAIN_FRACTION))) train_values = [warning[d] for d in dates[:split] if d in warning] warn_threshold = _percentile(train_values, WARN_PERCENTILE) if warn_threshold is None: return {"available": False, "reason": "insufficient warning history"} holdout_events = [index for index in all_event_indices if index >= split] fitted_alarms = alarm_episodes(warning, dates, warn_threshold, start_index=split) holdout_sessions = max(1, len(dates) - split) fitted_metrics = _score_rule( fitted_alarms, holdout_events, dates, horizon, holdout_sessions ) fitted_events = fitted_metrics.pop("per_event") reliability = _reliability(dates, split, backing, len(all_events), len(holdout_events)) # --- ablations and baselines, all on fixed thresholds ---------------- # Fitted thresholds are deliberately excluded here: a threshold fitted on the # full sample would have lookahead the shipped rule does not, and one fitted # on a training split could only be scored on the four holdout events. Fixed # constants keep every row on the same events over the same sessions. state_series = { session: row["state"] for session, row in rows.items() if row["state"] is not None } vix_indicator = { session: value for session in dates if (value := rms._value_asof(vix_series, session)) is not None } # The fundamental channel is categorical and never enters a score, so it is # compared as its own rule and as a confluence gate rather than tuned as a # weight. With an empty observation series both are unmeasurable, and say so. fundamental_alarms = adverse_episodes(rows, dates, evaluable_start) confluence_alarms = confluence_episodes(warning_alarms, rows, dates) # Coverage-matched denominators. These rules only existed on the sessions the # channel had usable context, so scoring them over the whole window would # report an exposure they never had -- and one day of coverage would render # as 0/10. fundamental_events = covered_events(evaluable_events, rows, dates, horizon) fundamental_sessions = eligible_sessions(rows, dates, evaluable_start) fundamental_measurable = len(fundamental_events) >= MIN_EVENTS_FOR_CONFIDENCE comparison = [ _rule_row( "fundamental_adverse", "Fundamental context turns adverse", "fundamental", "The third channel on its own: transitions into an adverse capex / " "earnings-reaction state, with no market input at all.", fundamental_alarms, fundamental_events, dates, horizon, fundamental_sessions, measurable=fundamental_measurable, ), _rule_row( "confluence", "Confluence: Warning crossing while adverse", "fundamental", "The shipped market crossing, kept only when the fundamental channel " "agrees. Answers whether requiring agreement buys precision, at what " "cost in recall.", confluence_alarms, fundamental_events, dates, horizon, fundamental_sessions, measurable=fundamental_measurable, ), _rule_row( "market_over_covered", "Quadrant alert, covered window only", "fundamental", "The shipped market rule scored on exactly the events, sessions and " "alarms the two rows above were scored on. Without it, any difference " "between them and the headline could be the window rather than the " "channel.", # Alarms are restricted to the covered window too: counting crossings # that fired when the channel had no context would compare the market # rule's full exposure against the channel's partial one. [ index for index in warning_alarms if index >= evaluable_start and bool((rows.get(dates[index]) or {}).get("fundamental_usable")) ], fundamental_events, dates, horizon, fundamental_sessions, measurable=fundamental_measurable, ), _rule_row( "quadrant_stress_entry", "Quadrant alert, both axes high", "ablation", "The same replay, recording only entries into the both-high quadrant. " "State is coincident by construction, so requiring it should convert " "leads into confirmations.", entry_alarms(fires, STRESS_QUADRANT), evaluable_events, dates, horizon, evaluable_sessions, ), _rule_row( "warning_bare", f"Warning >= {QUAD_Y_DIV:.0f} (bare crossing)", "ablation", "The shipped divider with none of the quadrant machinery: no State " "condition, no hysteresis, no confirmation, no cooldown.", alarm_episodes(warning, dates, QUAD_Y_DIV, start_index=evaluable_start), evaluable_events, dates, horizon, evaluable_sessions, ), _rule_row( "state_bare", f"State >= {QUAD_X_DIV:.0f} (bare crossing)", "ablation", "The coincident axis alone. State measures stress that has already " "arrived, so a competitive lead here would be surprising.", alarm_episodes(state_series, dates, QUAD_X_DIV, start_index=evaluable_start), evaluable_events, dates, horizon, evaluable_sessions, ), _rule_row( "smh_below_50dma", f"{leader} below its {BASELINE_SMA_WINDOW}-DMA", "baseline", "The crudest possible trend rule, and free.", alarm_episodes( below_average_series(benchmark, BASELINE_SMA_WINDOW), dates, 50.0, start_index=evaluable_start, ), evaluable_events, dates, horizon, evaluable_sessions, ), _rule_row( "vix_level", f"VIX >= {BASELINE_VIX_LEVEL:.0f}", "baseline", "The market's own risk gauge, unweighted and unmodelled.", alarm_episodes( vix_indicator, dates, BASELINE_VIX_LEVEL, start_index=evaluable_start ), evaluable_events, dates, horizon, evaluable_sessions, ), ] null_model = _null_model( len(warning_alarms), evaluable_events, dates, horizon, evaluable_start, shipped_metrics["events_warned"], # Passed rather than defaulted: a default argument binds the constant at # import, so overriding it (in tests) would silently do nothing. draws=NULL_DRAWS, seed=NULL_SEED, ) eras = _era_split( warning_alarms, evaluable_events, dates, horizon, evaluable_start, credit_from ) basket_asof = date.fromisoformat(config["basket_asof"]) retrospective = dates[evaluable_start] < basket_asof evaluation = "exploratory" if retrospective else "holdout" lead_text = ( f"median lead {shipped_metrics['median_lead_days']:.0f} sessions" if shipped_metrics["median_lead_days"] is not None else "no successful warning lead" ) summary = ( f"{evaluation.capitalize()} replay of the shipped quadrant alert over " f"{evaluable_sessions} sessions: it entered Warning-high territory ahead of " f"{shipped_metrics['events_warned']} of {shipped_metrics['events']} 10% " f"corrections, with {shipped_metrics['false_alarms_per_year']:.1f} false " f"alarms/year and {lead_text}. Its dividers are fixed constants rather than " f"fitted, so there is no training split and every detected correction is " f"evaluable — compare it against the ablations and baselines below before " f"reading the ratio as good or bad." ) report = { "available": True, "schema": STUDY_SCHEMA, "methodology": rms.METHODOLOGY, "generated_at": datetime.now(timezone.utc).isoformat(), "evaluation": evaluation, "summary": summary, "params": { "benchmark": leader, "outcome": "10% correction from trailing 52-week high", "event_threshold_pct": threshold_pct, "event_cooldown_days": EVENT_COOLDOWN_DAYS, "horizon_days": horizon, "credit_sensor_from": credit_from.isoformat() if credit_from else None, "basket_hash": rms._basket_hash(config["breadth_basket"]), "basket_asof": config["basket_asof"], }, # The channel's actual exposure, which is what its rows are scored on. # The series starts empty -- the observation lived in a single # overwritten settings slot until 2026-08-12 -- and it accumulates one # observation at a time, so for a long while these rows are unmeasurable # rather than unsuccessful. Stating the exposure is what stops the table # inventing a failed result out of a thin one. "fundamental_coverage": { "observations": len(observations), "sessions_eligible": fundamental_sessions, "evaluable_sessions": evaluable_sessions, "events_covered": len(fundamental_events), "events_evaluable": len(evaluable_events), "minimum_events": MIN_EVENTS_FOR_CONFIDENCE, "measurable": fundamental_measurable, }, "sample": { "start": dates[0].isoformat(), "end": dates[-1].isoformat(), "sessions": len(dates), # Not "test_start": the shipped rule fits nothing, so this is where # the baseline seeds and every rule becomes measurable, not where a # holdout begins. The fitted variant's split lives under "fitted". "evaluable_from": dates[evaluable_start].isoformat(), "evaluable_sessions": evaluable_sessions, "events_detected": len(all_events), "events_evaluable": len(evaluable_events), }, "shipped": { "rule": { "state_divider": QUAD_X_DIV, "warning_divider": QUAD_Y_DIV, "margin": QUAD_MARGIN, "confirm_sessions": 2, "cooldown_days": QUAD_COOLDOWN_DAYS, "entry": "Warning-high quadrant (early warning or active stress)", }, "metrics": shipped_metrics, "events": shipped_events, "quadrant_changes": len(fires), "fires": fires, "by_era": eras, }, "comparison": comparison, "null_model": null_model, "fitted": { "params": { "train_fraction": TRAIN_FRACTION, "warn_percentile": WARN_PERCENTILE, "warn_threshold": round(warn_threshold, 1), }, "sample": { "train_end": dates[split - 1].isoformat(), "test_start": dates[split].isoformat(), "holdout_sessions": holdout_sessions, }, "metrics": fitted_metrics, "events": fitted_events, }, "reliability": reliability, "recent_breadth": [ {"date": d.isoformat(), "breadth": breadth[d], "warning": warning.get(d)} for d in dates[-90:] if d in breadth ], } logger.info(json.dumps({ "event": "regime_event_study_complete", "evaluation": evaluation, "shipped_events": shipped_metrics["events"], "shipped_warned": shipped_metrics["events_warned"], "shipped_false_alarms_per_year": shipped_metrics["false_alarms_per_year"], "quadrant_changes": len(fires), "fitted_events": fitted_metrics["events"], "fitted_warned": fitted_metrics["events_warned"], "null_p_at_least_observed": (null_model or {}).get("p_at_least_observed"), "underpowered": reliability["underpowered"], "sensor_coverage_mismatch": reliability["sensor_coverage_mismatch"], })) return report async def run_and_store(db: AsyncSession) -> dict: report = await run_event_study(db) await update_setting(db, KEY_REPORT, json.dumps(report)) return report async def get_event_study_report(db: AsyncSession) -> dict | None: setting = await settings_store.get_setting(db, KEY_REPORT) if setting is None: return None try: report = json.loads(setting.value) except (TypeError, ValueError): return None if report.get("methodology") != rms.METHODOLOGY: return None # A pre-replay report parses fine and carries the current methodology, so the # shape has to be checked separately or the panel renders a headline-less v4. return report if report.get("schema") == STUDY_SCHEMA else None