feat: replace regime monitor with v2 methodology

This commit is contained in:
2026-07-15 09:02:56 +02:00
parent fd21067a40
commit 1d5b1489be
17 changed files with 1599 additions and 1535 deletions
+191 -239
View File
@@ -1,21 +1,9 @@
"""Event study: does a candidate indicator actually *lead* regime breaks?
"""Compact chronological validation for the Regime Monitor warning score.
This is a backtest-style measurement, but the unit of analysis is **events**
(historical drawdowns), not trades. For each candidate indicator it answers:
- how many days of warning did it give before the break (event-centered)?
- at what false-alarm cost (signal-centered precision/recall vs. the base rate)?
It compares the breadth-divergence early-warning candidate against a deterministic
**coincident** price composite (the existing regime price sub-scores), so you can
see whether the candidate crosses *earlier*. Everything is price/breadth only —
no LLM/FRED — so the result is reproducible.
Honest caveat: with only a handful of real drawdowns in ~5y, the sample is tiny
and the numbers are noisy. Read the median lead time as an order of magnitude, and
do NOT overfit thresholds to this history.
Report is cached in a SystemSetting (mirrors ``backtest_service``); a manual job
(Admin → Jobs) drives it.
The study calls its outcome a 10% correction, uses the first 70% of sessions to
freeze an 80th-percentile warning threshold, and reports alarm episodes only on
the final 30%. It is still labelled exploratory while the fixed breadth basket
is reconstructed before its freeze date.
"""
from __future__ import annotations
@@ -34,304 +22,268 @@ logger = logging.getLogger(__name__)
KEY_REPORT = "regime_event_study"
# Defaults. The 15% threshold gave only 2 events in 5y (statistically useless),
# so the default is lower with a cooldown-based dedup to surface more, cleaner
# events. Each indicator "warns" at its OWN 80th percentile rather than a shared
# absolute level, so the leading vs. coincident comparison is fair across scales.
EVENT_THRESHOLD_PCT = 10.0 # drawdown from the 52w high that counts as a "break"
COOLDOWN_DAYS = 40 # min trading days between event onsets (dedup)
DRAWDOWN_LOOKBACK = 252 # 52-week trailing high
HORIZON_DAYS = 20 # signal-centered prediction horizon
WARN_PERCENTILE = 80.0 # each indicator warns at its own Nth percentile
PRE, POST = 60, 20 # event-centered window (trading days)
EVENT_THRESHOLD_PCT = 10.0
EVENT_COOLDOWN_DAYS = 40
DRAWDOWN_LOOKBACK = 252
HORIZON_DAYS = 20
WARN_PERCENTILE = 80.0
TRAIN_FRACTION = 0.70
def _median(values: list[float]) -> float | None:
if not values:
return None
s = sorted(values)
n = len(s)
mid = n // 2
return float(s[mid]) if n % 2 else (s[mid - 1] + s[mid]) / 2.0
ordered = sorted(values)
middle = len(ordered) // 2
return (
float(ordered[middle])
if len(ordered) % 2
else (ordered[middle - 1] + ordered[middle]) / 2.0
)
def _percentile(values: list[float], pct: float) -> float | None:
"""Linear-interpolated percentile of the non-None values."""
vals = sorted(v for v in values if v is not None)
if not vals:
ordered = sorted(v for v in values if v is not None)
if not ordered:
return None
k = (len(vals) - 1) * (pct / 100.0)
lo = int(k)
hi = min(lo + 1, len(vals) - 1)
return vals[lo] + (vals[hi] - vals[lo]) * (k - lo)
position = (len(ordered) - 1) * pct / 100.0
lower = int(position)
upper = min(lower + 1, len(ordered) - 1)
return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower)
# ---------------------------------------------------------------------------
# Event detection
# ---------------------------------------------------------------------------
def detect_events(
closes: list[float],
dates: list[date],
threshold_pct: float = EVENT_THRESHOLD_PCT,
lookback: int = DRAWDOWN_LOOKBACK,
cooldown: int = COOLDOWN_DAYS,
cooldown: int = EVENT_COOLDOWN_DAYS,
) -> list[dict]:
"""Drawdown events: ``t0`` = a day the drawdown from the trailing 52w high
crosses up through ``threshold_pct`` (rising edge). De-duplicated by a
``cooldown`` of trading days, so a continuous decline counts once but distinct
drawdowns separated by a recovery each register."""
"""Rising-edge corrections from the trailing 52-week high."""
events: list[dict] = []
prev_dd = 0.0
previous_drawdown = 0.0
last_event = -10**9
for i in range(len(closes)):
window = closes[max(0, i - lookback + 1): i + 1]
hi = max(window)
dd = (hi - closes[i]) / hi * 100.0 if hi > 0 else 0.0
if dd >= threshold_pct and prev_dd < threshold_pct and (i - last_event) >= cooldown:
events.append({"date": dates[i].isoformat(), "index": i, "depth_pct": round(dd, 1)})
last_event = i
prev_dd = dd
for index, close in enumerate(closes):
high = max(closes[max(0, index - lookback + 1): index + 1])
drawdown = (high - close) / high * 100.0 if high > 0 else 0.0
if (
drawdown >= threshold_pct
and previous_drawdown < threshold_pct
and index - last_event >= cooldown
):
events.append({
"date": dates[index].isoformat(),
"index": index,
"depth_pct": round(drawdown, 1),
})
last_event = index
previous_drawdown = drawdown
return events
# ---------------------------------------------------------------------------
# Event-centered: lead time + mean path
# ---------------------------------------------------------------------------
def _lead(indicator: dict[date, float], t0: int, dates: list[date], pre: int, threshold: float) -> int | None:
"""Earliest day within ``[t0-pre, t0]`` at which the indicator crosses
``threshold`` — i.e. how many days of warning before the event, or None."""
lead: int | None = None
for k in range(0, pre + 1):
idx = t0 - k
if idx < 0:
break
v = indicator.get(dates[idx])
if v is not None and v >= threshold:
lead = k # keep going: the largest k = earliest warning in the window
return lead
def event_centered(
def alarm_episodes(
indicator: dict[date, float],
events_idx: list[int],
dates: list[date],
pre: int = PRE,
post: int = POST,
threshold: float = 60.0,
) -> dict:
"""Align the indicator at each event's ``t0`` and measure how early it warned.
threshold: float,
start_index: int = 1,
) -> list[int]:
"""Indices where the warning crosses upward; it must reset below first."""
alarms: list[int] = []
was_high = False
if start_index > 0:
previous = indicator.get(dates[start_index - 1])
was_high = previous is not None and previous >= threshold
for index in range(start_index, len(dates)):
value = indicator.get(dates[index])
if value is None:
continue
high = value >= threshold
if high and not was_high:
alarms.append(index)
was_high = high
return alarms
Lead time is measured against ``threshold`` (each indicator gets its own,
derived from its distribution). Also returns the cross-event mean path.
"""
def evaluate_alarms(
alarm_indices: list[int],
event_indices: list[int],
dates: list[date],
horizon: int = HORIZON_DAYS,
) -> dict:
"""Event recall, episode false alarms, and lead time for one holdout."""
leads: list[float] = []
sums: dict[int, float] = {}
counts: dict[int, int] = {}
for t0 in events_idx:
lead = _lead(indicator, t0, dates, pre, threshold)
per_event: list[dict] = []
warned = 0
for event_index in event_indices:
matching = [
alarm for alarm in alarm_indices if 0 < event_index - alarm <= horizon
]
lead = max((event_index - alarm for alarm in matching), default=None)
if lead is not None:
leads.append(lead)
for rel in range(-pre, post + 1):
idx = t0 + rel
if 0 <= idx < len(dates):
v = indicator.get(dates[idx])
if v is not None:
sums[rel] = sums.get(rel, 0.0) + v
counts[rel] = counts.get(rel, 0) + 1
mean_path = [
{"rel_day": rel, "value": round(sums[rel] / counts[rel], 1)} for rel in sorted(sums)
]
warned += 1
leads.append(float(lead))
per_event.append({
"date": dates[event_index].isoformat(),
"warned": lead is not None,
"lead_days": lead,
})
false_alarms = sum(
1
for alarm in alarm_indices
if not any(0 < event - alarm <= horizon for event in event_indices)
)
return {
"events": len(event_indices),
"events_warned": warned,
"events_missed": len(event_indices) - warned,
"alarm_episodes": len(alarm_indices),
"false_alarms": false_alarms,
"median_lead_days": _median(leads),
"events_with_signal": len(leads),
"events_total": len(events_idx),
"warn_threshold": round(threshold, 1),
"mean_path": mean_path,
"per_event": per_event,
}
# ---------------------------------------------------------------------------
# Signal-centered: precision / recall vs. base rate
# ---------------------------------------------------------------------------
def signal_centered(
indicator: dict[date, float],
events_idx: list[int],
def _warning_series(
prices: dict[str, rms.Series],
breadth_divergence: dict[date, float],
dates: list[date],
horizon: int = HORIZON_DAYS,
thresholds: list[float] | None = None,
) -> dict:
"""Treat ``indicator >= threshold`` as predicting a break within ``horizon``
days. Sweep thresholds → precision/recall/alarm count, plus the base rate."""
thresholds = thresholds or [50, 55, 60, 65, 70, 75, 80]
n = len(dates)
labels = [1 if any(i < e <= i + horizon for e in events_idx) else 0 for i in range(n)]
positives = sum(labels)
base_rate = positives / n if n else 0.0
rows: list[dict] = []
for th in thresholds:
tp = fp = fn = 0
for i in range(n):
v = indicator.get(dates[i])
if v is None:
continue
pred = v >= th
if pred and labels[i]:
tp += 1
elif pred and not labels[i]:
fp += 1
elif not pred and labels[i]:
fn += 1
precision = tp / (tp + fp) if (tp + fp) else None
recall = tp / (tp + fn) if (tp + fn) else None
rows.append({
"threshold": th,
"precision": round(precision, 3) if precision is not None else None,
"recall": round(recall, 3) if recall is not None else None,
"alarms": tp + fp,
})
return {"base_rate": round(base_rate, 3), "horizon_days": horizon, "rows": rows}
# ---------------------------------------------------------------------------
# Coincident baseline (deterministic price composite, reusing the regime sub-scores)
# ---------------------------------------------------------------------------
def _coincident_series(prices: dict[str, list], dates: list[date], config: dict) -> dict[date, float]:
"""Mean of the available price sub-scores (P1-P4) as-of each date — the
coincident baseline the leading candidate must beat on lead time."""
lw = float(config.get("leader_weight", 2.0))
lb = int(config.get("rs_lookback", 60))
t = config["tickers"]
smh_full = prices.get(t["leaders"][0], []) if t["leaders"] else []
qqq_full = prices.get(t["confirm"][0], []) if t["confirm"] else []
spy_full = prices.get(t["market"], [])
config: dict,
) -> dict[date, float]:
"""Technical Warning score used historically (fundamentals have no PIT history)."""
tickers = config["tickers"]
smh_full = prices.get(tickers["leaders"][0], [])
spy_full = prices.get(tickers["market"], [])
out: dict[date, float] = {}
for d in dates:
smh = rms._closes_asof(smh_full, d)
qqq = rms._closes_asof(qqq_full, d)
spy = rms._closes_asof(spy_full, d)
subs = [
rms.p1_trend_break(smh, qqq, lw),
rms.p2_death_cross(smh, qqq, lw),
rms.p3_drawdown(smh, qqq),
rms.p4_relative_strength(smh, spy, lb),
]
vals = [v for v in subs if v is not None]
if vals:
out[d] = round(sum(vals) / len(vals), 2)
for session in dates:
divergence = breadth_divergence.get(session)
relative = rms.p4_relative_strength(
rms._closes_asof(smh_full, session),
rms._closes_asof(spy_full, session),
)
values: list[tuple[float, float]] = []
if divergence is not None:
values.append((divergence, rms.WARNING_WEIGHTS["breadth_divergence"]))
if relative is not None:
values.append((relative, rms.WARNING_WEIGHTS["relative_strength"]))
if values:
out[session] = round(
sum(value * weight for value, weight in values)
/ sum(weight for _, weight in values),
2,
)
return out
# ---------------------------------------------------------------------------
# Orchestration
# ---------------------------------------------------------------------------
async def run_event_study(
db: AsyncSession,
threshold_pct: float = EVENT_THRESHOLD_PCT,
horizon: int = HORIZON_DAYS,
cooldown: int = COOLDOWN_DAYS,
warn_percentile: float = WARN_PERCENTILE,
) -> dict:
"""Run the study: detect events on the benchmark, then measure breadth-divergence
vs. the coincident price composite. Best-effort; returns available=False on no data."""
config = await rms.get_regime_config(db)
end = date.today()
start = end - timedelta(days=5 * 365 + 30)
prices = await rms._fetch_prices(config, start, end)
leader = config["tickers"]["leaders"][0] if config["tickers"]["leaders"] else "SMH"
bench = sorted(prices.get(leader, []), key=lambda x: x[0])
if len(bench) < 260:
leader = config["tickers"]["leaders"][0]
benchmark = sorted(prices.get(leader, []), key=lambda item: item[0])
if len(benchmark) < 500:
return {"available": False, "reason": "insufficient benchmark history"}
dates = [d for d, _ in bench]
closes = [c for _, c in bench]
events = detect_events(closes, dates, threshold_pct, cooldown=cooldown)
events_idx = [e["index"] for e in events]
dates = [d for d, _ in benchmark]
closes = [value for _, value in benchmark]
breadth, _ = await breadth_service.compute_breadth_details(
db, config["breadth_basket"], window=200, min_tickers=20
)
divergence = breadth_service.compute_divergence_series(breadth, benchmark)
warning = _warning_series(prices, divergence, dates, config)
breadth = await breadth_service.compute_breadth_series(db)
divergence = breadth_service.compute_divergence_series(breadth, bench)
coincident = _coincident_series(prices, dates, config)
split = max(1, min(len(dates) - 1, int(len(dates) * TRAIN_FRACTION)))
train_values = [warning[d] for d in dates[:split] if d in warning]
warn_threshold = _percentile(train_values, WARN_PERCENTILE)
if warn_threshold is None:
return {"available": False, "reason": "insufficient warning history"}
# Each indicator warns at its OWN distribution's percentile, so a leading
# indicator isn't penalised for living on a different scale than the baseline.
warn = {
"breadth_divergence": _percentile(list(divergence.values()), warn_percentile) or 60.0,
"coincident_price": _percentile(list(coincident.values()), warn_percentile) or 60.0,
}
series_by_key = {"breadth_divergence": divergence, "coincident_price": coincident}
all_events = detect_events(closes, dates, threshold_pct)
holdout_events = [event["index"] for event in all_events if event["index"] >= split]
alarms = alarm_episodes(warning, dates, warn_threshold, start_index=split)
metrics = evaluate_alarms(alarms, holdout_events, dates, horizon)
holdout_sessions = max(1, len(dates) - split)
metrics["false_alarms_per_year"] = round(
metrics["false_alarms"] / (holdout_sessions / 252.0), 2
)
def _evaluate(series: dict[date, float], threshold: float) -> dict:
return {
**event_centered(series, events_idx, dates, threshold=threshold),
"signal": signal_centered(series, events_idx, dates, horizon),
}
indicators = {key: _evaluate(series_by_key[key], warn[key]) for key in series_by_key}
# Per-event comparison: which event, and each indicator's lead on THAT event —
# so a median over a tiny sample can't hide an apples-to-oranges comparison.
per_event = [
{
"date": e["date"],
"depth_pct": e["depth_pct"],
"breadth_lead": _lead(divergence, e["index"], dates, PRE, warn["breadth_divergence"]),
"coincident_lead": _lead(coincident, e["index"], dates, PRE, warn["coincident_price"]),
}
for e in events
]
bd = indicators["breadth_divergence"]["median_lead_days"]
cd = indicators["coincident_price"]["median_lead_days"]
lead_delta = (bd - cd) if (bd is not None and cd is not None) else None
recent_breadth = [
{"date": d.isoformat(), "breadth": breadth[d], "divergence": divergence.get(d)}
for d in dates[-90:]
if d in breadth
]
basket_asof = date.fromisoformat(config["basket_asof"])
retrospective = dates[split] < basket_asof
evaluation = "exploratory" if retrospective else "holdout"
lead_text = (
f"median lead {metrics['median_lead_days']:.0f} sessions"
if metrics["median_lead_days"] is not None
else "no successful warning lead"
)
summary = (
f"{evaluation.capitalize()} chronological test: warning episodes preceded "
f"{metrics['events_warned']}/{metrics['events']} 10% corrections; "
f"{metrics['events_missed']} missed, {metrics['false_alarms_per_year']:.1f} "
f"false alarms/year, {lead_text}."
)
per_event = metrics.pop("per_event")
report = {
"available": True,
"methodology": rms.METHODOLOGY,
"generated_at": datetime.now(timezone.utc).isoformat(),
"evaluation": evaluation,
"summary": summary,
"params": {
"benchmark": leader,
"outcome": "10% correction from trailing 52-week high",
"event_threshold_pct": threshold_pct,
"cooldown_days": cooldown,
"event_cooldown_days": EVENT_COOLDOWN_DAYS,
"horizon_days": horizon,
"warn_percentile": warn_percentile,
"train_fraction": TRAIN_FRACTION,
"warn_percentile": WARN_PERCENTILE,
"warn_threshold": round(warn_threshold, 1),
"basket_hash": rms._basket_hash(config["breadth_basket"]),
"basket_asof": config["basket_asof"],
},
"events": events,
"indicators": indicators,
"per_event": per_event,
"lead_delta_days": lead_delta,
"recent_breadth": recent_breadth,
"sample": {
"start": dates[0].isoformat(),
"end": dates[-1].isoformat(),
"train_end": dates[split - 1].isoformat(),
"test_start": dates[split].isoformat(),
"sessions": len(dates),
"holdout_sessions": holdout_sessions,
},
"metrics": metrics,
"events": per_event,
"recent_breadth": [
{"date": d.isoformat(), "breadth": breadth[d], "warning": warning.get(d)}
for d in dates[-90:]
if d in breadth
],
}
logger.info(json.dumps({
"event": "event_study_complete", "events": len(events),
"breadth_lead": bd, "coincident_lead": cd,
"event": "regime_event_study_complete",
"evaluation": evaluation,
"events": metrics["events"],
"warned": metrics["events_warned"],
"false_alarms_per_year": metrics["false_alarms_per_year"],
}))
return report
async def run_and_store(db: AsyncSession) -> dict:
"""Run the event study and cache the report in a SystemSetting. Job entrypoint."""
report = await run_event_study(db)
await update_setting(db, KEY_REPORT, json.dumps(report))
return report
async def get_event_study_report(db: AsyncSession) -> dict | None:
"""Return the last cached event-study report, or None if never run."""
setting = await settings_store.get_setting(db, KEY_REPORT)
if setting is None:
return None
try:
return json.loads(setting.value)
report = json.loads(setting.value)
except (TypeError, ValueError):
return None
return report if report.get("methodology") == rms.METHODOLOGY else None