dennisthiessen
66444af65c
Plots the index, early-warning, and combined scores over time beneath the live gauges, with a 1M/3M/6M/All range toggle and band reference lines — so the trend and any divergence between the scores is visible, not just today's snapshot. - Backend: GET /regime/history + get_regime_history (the three scores per snapshot date from regime_snapshots). - Frontend: recharts line chart, lazy-loaded so recharts ships in its own regime-tab chunk instead of nearly doubling the main bundle. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
854 lines
32 KiB
Python
854 lines
32 KiB
Python
"""AI/Tech Regime-Change Monitor.
|
|
|
|
A standalone, observational tool: it scores how far the AI/Tech bull regime has
|
|
deteriorated toward a re-rating, as a single 0-100 **index** (not a calibrated
|
|
probability), broken down per signal. It is intentionally decoupled — nothing
|
|
here feeds gates, scoring, alerts, or trade logic. It only computes a number for
|
|
its own tab.
|
|
|
|
Design mirrors ``market_regime_service``: benchmark/sector bars are pulled
|
|
directly via Alpaca (no Universe membership needed), macro inputs (VIX, HY credit
|
|
spreads) come from FRED, and the daily result is persisted as one
|
|
``RegimeSnapshot`` row per date so the UI can show a 7/30-day trend. On the first
|
|
run the history is backfilled by replaying the (already-fetched) price/FRED series
|
|
as-of each past day, so the trend is populated immediately.
|
|
|
|
Signals (sub-score 0 = healthy … 100 = regime breaking):
|
|
P1 trend break (% under 200-DMA, SMH-led) P2 death cross + 200-slope
|
|
P3 drawdown from 52w high P4 relative strength SMH/SPY
|
|
P5 volatility (VIX) P6 NVDA canary divergence (opt.)
|
|
F1 hyperscaler capex guidance (LLM/manual) F2 HY credit-spread percentile
|
|
F3 "good news, stock down" (LLM/manual) F4 market breadth RSP/SPY
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import copy
|
|
import json
|
|
import logging
|
|
import os
|
|
from datetime import date, datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.config import settings
|
|
from app.exceptions import ProviderError
|
|
from app.models.regime_snapshot import RegimeSnapshot
|
|
from app.providers.alpaca import AlpacaOHLCVProvider
|
|
from app.services import breadth_service, settings_store
|
|
from app.services.admin_service import update_setting
|
|
from app.services.sentiment_provider_service import _resolve as resolve_llm_config
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
|
|
|
|
KEY_CONFIG = "regime_monitor_config"
|
|
KEY_FUNDAMENTALS = "regime_fundamental_overrides"
|
|
|
|
# All weights/thresholds are admin-editable via the KEY_CONFIG SystemSetting.
|
|
# Default weights sum to 100 (P6 off). SMH is the leading sensor, QQQ confirms.
|
|
DEFAULT_CONFIG: dict = {
|
|
"tickers": {
|
|
"leaders": ["SMH"], # semis — fast early signal
|
|
"confirm": ["QQQ"], # broad tech — confirmation
|
|
"market": "SPY",
|
|
"breadth": "RSP", # equal-weight S&P for breadth
|
|
"canary": "NVDA", # sector lead-stock (optional early warning)
|
|
"hyperscalers": ["GOOGL", "AMZN", "META", "MSFT"],
|
|
},
|
|
"weights": {
|
|
"P1": 12, "P2": 8, "P3": 10, "P4": 8, "P5": 7, "P6": 0,
|
|
"F1": 25, "F2": 15, "F3": 8, "F4": 7,
|
|
},
|
|
"alert_threshold": 65,
|
|
# Observational early-warning blend: a small Combined score = weighted mean of
|
|
# the coincident index and the breadth-divergence early-warning score. Kept
|
|
# separate from the index weights above so the early-warning side stays
|
|
# decoupled until proven. Tunable; need not sum to 1 (normalised).
|
|
"combined_weights": {"coincident": 0.6, "early_warning": 0.4},
|
|
"leader_weight": 2.0, # SMH counts 2x vs QQQ where both feed a signal
|
|
"rs_lookback": 60, # trading days for relative-strength / breadth trend
|
|
"fundamental_staleness_days": 80,
|
|
}
|
|
|
|
SIGNAL_LABELS: dict[str, str] = {
|
|
"P1": "Trend break (200-DMA)",
|
|
"P2": "Death cross + slope",
|
|
"P3": "Drawdown from 52w high",
|
|
"P4": "Relative strength SMH/SPY",
|
|
"P5": "Volatility (VIX)",
|
|
"P6": "NVDA canary divergence",
|
|
"F1": "Hyperscaler capex guidance",
|
|
"F2": "Credit spreads (HY OAS)",
|
|
"F3": "Good news, stock down",
|
|
"F4": "Market breadth RSP/SPY",
|
|
}
|
|
|
|
_PRICE_SIGNALS = {"P1", "P2", "P3", "P4", "P6"}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Small numeric helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _clamp(x: float, lo: float = 0.0, hi: float = 100.0) -> float:
|
|
return max(lo, min(hi, x))
|
|
|
|
|
|
def _sma(values: list[float], window: int) -> float | None:
|
|
if len(values) < window:
|
|
return None
|
|
return sum(values[-window:]) / window
|
|
|
|
|
|
def _mean(values: list[float]) -> float | None:
|
|
return sum(values) / len(values) if values else None
|
|
|
|
|
|
def _blend(leader: float | None, confirm: float | None, leader_weight: float) -> float | None:
|
|
"""Weighted blend of a leading vs a confirming sub-score (SMH vs QQQ)."""
|
|
parts: list[tuple[float, float]] = []
|
|
if leader is not None:
|
|
parts.append((leader, leader_weight))
|
|
if confirm is not None:
|
|
parts.append((confirm, 1.0))
|
|
if not parts:
|
|
return None
|
|
num = sum(v * w for v, w in parts)
|
|
den = sum(w for _, w in parts)
|
|
return num / den
|
|
|
|
|
|
def band_for(score: float) -> str:
|
|
"""Map the 0-100 index onto its label band."""
|
|
if score < 30:
|
|
return "stable"
|
|
if score < 60:
|
|
return "watch"
|
|
if score < 80:
|
|
return "elevated"
|
|
return "breaking"
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Pure sub-score functions (0 = healthy, 100 = regime breaking). None = no data.
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _under_200(closes: list[float]) -> float | None:
|
|
sma200 = _sma(closes, 200)
|
|
if sma200 is None:
|
|
return None
|
|
return 100.0 if closes[-1] < sma200 else 0.0
|
|
|
|
|
|
def p1_trend_break(smh: list[float], qqq: list[float], leader_weight: float) -> float | None:
|
|
"""Weighted share trading below the 200-DMA (SMH leads, QQQ confirms)."""
|
|
return _blend(_under_200(smh), _under_200(qqq), leader_weight)
|
|
|
|
|
|
def _death_cross(closes: list[float]) -> float | None:
|
|
sma50 = _sma(closes, 50)
|
|
sma200 = _sma(closes, 200)
|
|
if sma50 is None or sma200 is None or len(closes) < 221 or sma200 == 0:
|
|
return None
|
|
gap_pct = (sma50 / sma200 - 1.0) * 100.0
|
|
severity = 0.0 if gap_pct >= 0 else _clamp(-gap_pct * 20.0) # -5% gap -> 100
|
|
sma200_past = _sma(closes[:-20], 200)
|
|
slope_factor = 1.0
|
|
if sma200_past:
|
|
slope_pct = (sma200 / sma200_past - 1.0) * 100.0
|
|
slope_factor = 1.0 if slope_pct < 0 else 0.5 # damp if 200 still rising
|
|
return severity * slope_factor
|
|
|
|
|
|
def p2_death_cross(smh: list[float], qqq: list[float], leader_weight: float) -> float | None:
|
|
return _blend(_death_cross(smh), _death_cross(qqq), leader_weight)
|
|
|
|
|
|
def _drawdown(closes: list[float]) -> float | None:
|
|
if len(closes) < 30:
|
|
return None
|
|
window = closes[-252:]
|
|
peak = max(window)
|
|
if peak <= 0:
|
|
return None
|
|
dd_pct = (peak - closes[-1]) / peak * 100.0
|
|
return _clamp(dd_pct * 5.0) # 20% below 52w high -> 100
|
|
|
|
|
|
def p3_drawdown(smh: list[float], qqq: list[float]) -> float | None:
|
|
vals = [v for v in (_drawdown(smh), _drawdown(qqq)) if v is not None]
|
|
return max(vals) if vals else None
|
|
|
|
|
|
def _ratio_trend(a: list[float], b: list[float], lookback: int) -> float | None:
|
|
"""Falling a/b (a underperforming b) -> higher score. Flat -> 50."""
|
|
if len(a) < lookback + 1 or len(b) < lookback + 1:
|
|
return None
|
|
if b[-1] == 0 or b[-lookback - 1] == 0:
|
|
return None
|
|
now = a[-1] / b[-1]
|
|
past = a[-lookback - 1] / b[-lookback - 1]
|
|
if past == 0:
|
|
return None
|
|
chg_pct = (now / past - 1.0) * 100.0
|
|
return _clamp(50.0 - chg_pct * 5.0) # -10% -> 100, +10% -> 0
|
|
|
|
|
|
def p4_relative_strength(smh: list[float], spy: list[float], lookback: int) -> float | None:
|
|
return _ratio_trend(smh, spy, lookback)
|
|
|
|
|
|
def f4_breadth(rsp: list[float], spy: list[float], lookback: int) -> float | None:
|
|
"""Narrowing breadth (equal-weight lagging cap-weight) -> RSP/SPY falls -> higher."""
|
|
return _ratio_trend(rsp, spy, lookback)
|
|
|
|
|
|
def p5_volatility(vix: float | None) -> float | None:
|
|
if vix is None:
|
|
return None
|
|
return _clamp((vix - 15.0) / 15.0 * 100.0) # <=15 -> 0, >=30 -> 100
|
|
|
|
|
|
def f2_credit_spreads(oas_values: list[float]) -> float | None:
|
|
"""Percentile rank of the latest HY OAS within the window. Wider = higher."""
|
|
if len(oas_values) < 30:
|
|
return None
|
|
latest = oas_values[-1]
|
|
below = sum(1 for v in oas_values if v <= latest)
|
|
return _clamp(below / len(oas_values) * 100.0)
|
|
|
|
|
|
def p6_canary(nvda: list[float], smh: list[float]) -> float | None:
|
|
"""NVDA below its 50-DMA while SMH's trend is still intact = lead divergence."""
|
|
sma50 = _sma(nvda, 50)
|
|
if sma50 is None:
|
|
return None
|
|
nvda_weak = nvda[-1] < sma50
|
|
sma200 = _sma(smh, 200)
|
|
smh_intact = sma200 is not None and smh[-1] > sma200
|
|
if nvda_weak and smh_intact:
|
|
return 100.0
|
|
return 50.0 if nvda_weak else 0.0
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Aggregation
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def compute_regime_score(sub_scores: dict[str, float | None], weights: dict[str, float]) -> dict:
|
|
"""Weighted mean over the *available* signals (weight>0 and score present).
|
|
|
|
Missing-data signals drop out of both numerator and denominator and are
|
|
reported with ``available=False``. Contributions sum to the total.
|
|
"""
|
|
denom = sum(
|
|
weights.get(sid, 0)
|
|
for sid, score in sub_scores.items()
|
|
if score is not None and weights.get(sid, 0) > 0
|
|
)
|
|
total = 0.0
|
|
breakdown: list[dict] = []
|
|
for sid in SIGNAL_LABELS:
|
|
weight = weights.get(sid, 0)
|
|
if weight <= 0:
|
|
continue
|
|
score = sub_scores.get(sid)
|
|
available = score is not None
|
|
contribution = (score * weight / denom) if (available and denom > 0) else 0.0
|
|
if available:
|
|
total += contribution
|
|
breakdown.append({
|
|
"id": sid,
|
|
"label": SIGNAL_LABELS[sid],
|
|
"sub_score": round(score, 1) if available else None,
|
|
"weight": weight,
|
|
"available": available,
|
|
"contribution": round(contribution, 2),
|
|
})
|
|
return {"total_score": round(total, 1), "band": band_for(total), "breakdown": breakdown}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# As-of series helpers (for backfill replay)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
Series = list[tuple[date, float]]
|
|
|
|
|
|
def _closes_asof(series: Series, as_of: date) -> list[float]:
|
|
return [v for d, v in series if d <= as_of]
|
|
|
|
|
|
def _value_asof(series: Series | None, as_of: date) -> float | None:
|
|
if not series:
|
|
return None
|
|
vals = [v for d, v in series if d <= as_of]
|
|
return vals[-1] if vals else None
|
|
|
|
|
|
def _window_asof(series: Series | None, as_of: date, years: float) -> list[float]:
|
|
if not series:
|
|
return []
|
|
start = as_of - timedelta(days=int(365 * years))
|
|
return [v for d, v in series if start <= d <= as_of]
|
|
|
|
|
|
def _compute_index(
|
|
prices: dict[str, Series],
|
|
vix_series: Series | None,
|
|
oas_series: Series | None,
|
|
overrides: dict,
|
|
config: dict,
|
|
as_of: date,
|
|
) -> dict:
|
|
"""Compute the full index result as-of *as_of* from raw series."""
|
|
t = config["tickers"]
|
|
lw = float(config.get("leader_weight", 2.0))
|
|
lb = int(config.get("rs_lookback", 60))
|
|
|
|
smh = _closes_asof(prices.get(t["leaders"][0], []), as_of) if t["leaders"] else []
|
|
qqq = _closes_asof(prices.get(t["confirm"][0], []), as_of) if t["confirm"] else []
|
|
spy = _closes_asof(prices.get(t["market"], []), as_of)
|
|
rsp = _closes_asof(prices.get(t["breadth"], []), as_of)
|
|
nvda = _closes_asof(prices.get(t["canary"], []), as_of)
|
|
vix = _value_asof(vix_series, as_of)
|
|
oas = _window_asof(oas_series, as_of, 3)
|
|
|
|
sub_scores: dict[str, float | None] = {
|
|
"P1": p1_trend_break(smh, qqq, lw),
|
|
"P2": p2_death_cross(smh, qqq, lw),
|
|
"P3": p3_drawdown(smh, qqq),
|
|
"P4": p4_relative_strength(smh, spy, lb),
|
|
"P5": p5_volatility(vix),
|
|
"P6": p6_canary(nvda, smh),
|
|
"F1": overrides.get("f1_score"),
|
|
"F2": f2_credit_spreads(oas),
|
|
"F3": overrides.get("f3_score"),
|
|
"F4": f4_breadth(rsp, spy, lb),
|
|
}
|
|
|
|
result = compute_regime_score(sub_scores, config["weights"])
|
|
result["date"] = as_of.isoformat()
|
|
result["alert_threshold"] = config.get("alert_threshold", 65)
|
|
result["inputs"] = {
|
|
"vix": round(vix, 2) if vix is not None else None,
|
|
"hy_oas": round(oas[-1], 2) if oas else None,
|
|
"fundamentals_fetched_at": overrides.get("fetched_at"),
|
|
}
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config + fundamental-override storage
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def get_regime_config(db: AsyncSession) -> dict:
|
|
"""DEFAULT_CONFIG deep-merged with the stored override (nested for dicts)."""
|
|
cfg = copy.deepcopy(DEFAULT_CONFIG)
|
|
raw = await settings_store.get_value(db, KEY_CONFIG)
|
|
if raw:
|
|
try:
|
|
stored = json.loads(raw)
|
|
for k, v in stored.items():
|
|
if isinstance(v, dict) and isinstance(cfg.get(k), dict):
|
|
cfg[k].update(v)
|
|
else:
|
|
cfg[k] = v
|
|
except (TypeError, ValueError):
|
|
logger.warning("Corrupt %s; using defaults", KEY_CONFIG)
|
|
return cfg
|
|
|
|
|
|
async def update_regime_config(db: AsyncSession, updates: dict) -> dict:
|
|
"""Merge *updates* into the stored config and persist. Returns the new config."""
|
|
cfg = await get_regime_config(db)
|
|
for k, v in (updates or {}).items():
|
|
if isinstance(v, dict) and isinstance(cfg.get(k), dict):
|
|
cfg[k].update(v)
|
|
else:
|
|
cfg[k] = v
|
|
await update_setting(db, KEY_CONFIG, json.dumps(cfg))
|
|
return cfg
|
|
|
|
|
|
async def get_fundamental_overrides(db: AsyncSession) -> dict:
|
|
"""Current F1/F3 override (LLM-proposed or manual). Defaults to neutral 50."""
|
|
raw = await settings_store.get_value(db, KEY_FUNDAMENTALS)
|
|
default = {"f1_score": 50.0, "f3_score": 50.0, "locked": False,
|
|
"reasoning": None, "fetched_at": None, "source": "default"}
|
|
if not raw:
|
|
return default
|
|
try:
|
|
stored = json.loads(raw)
|
|
except (TypeError, ValueError):
|
|
return default
|
|
return {**default, **stored}
|
|
|
|
|
|
async def set_fundamental_overrides(
|
|
db: AsyncSession,
|
|
f1_score: float | None = None,
|
|
f3_score: float | None = None,
|
|
locked: bool | None = None,
|
|
) -> dict:
|
|
"""Manual override of F1/F3. Setting any value locks out the LLM refresh
|
|
unless ``locked`` is explicitly cleared."""
|
|
current = await get_fundamental_overrides(db)
|
|
if f1_score is not None:
|
|
current["f1_score"] = _clamp(float(f1_score))
|
|
if f3_score is not None:
|
|
current["f3_score"] = _clamp(float(f3_score))
|
|
if locked is not None:
|
|
current["locked"] = bool(locked)
|
|
elif f1_score is not None or f3_score is not None:
|
|
current["locked"] = True
|
|
current["source"] = "manual"
|
|
current["fetched_at"] = datetime.now(timezone.utc).isoformat()
|
|
await update_setting(db, KEY_FUNDAMENTALS, json.dumps(current))
|
|
return current
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Data fetching: Alpaca prices + FRED macro
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _price_symbols(config: dict) -> list[str]:
|
|
t = config["tickers"]
|
|
syms = list(t["leaders"]) + list(t["confirm"]) + [t["market"], t["breadth"], t["canary"]]
|
|
seen: list[str] = []
|
|
for s in syms:
|
|
if s and s not in seen:
|
|
seen.append(s)
|
|
return seen
|
|
|
|
|
|
async def _fetch_prices(config: dict, start: date, end: date) -> dict[str, Series]:
|
|
if not settings.alpaca_api_key or not settings.alpaca_api_secret:
|
|
return {}
|
|
provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
|
|
out: dict[str, Series] = {}
|
|
for sym in _price_symbols(config):
|
|
try:
|
|
bars = await provider.fetch_ohlcv(sym, start, end)
|
|
out[sym] = sorted(((b.date, float(b.close)) for b in bars), key=lambda x: x[0])
|
|
except Exception as exc:
|
|
logger.warning("Regime monitor: price fetch failed for %s: %s", sym, exc)
|
|
return out
|
|
|
|
|
|
async def _fetch_fred_series(series_id: str, start: date, end: date) -> Series | None:
|
|
"""Fetch a FRED series as [(date, value)]. None if no API key configured."""
|
|
if not settings.fred_api_key:
|
|
return None
|
|
verify = _CA_BUNDLE if (_CA_BUNDLE and Path(_CA_BUNDLE).exists()) else True
|
|
params = {
|
|
"series_id": series_id,
|
|
"api_key": settings.fred_api_key,
|
|
"file_type": "json",
|
|
"observation_start": start.isoformat(),
|
|
"observation_end": end.isoformat(),
|
|
}
|
|
try:
|
|
async with httpx.AsyncClient(timeout=30, verify=verify) as client:
|
|
resp = await client.get(
|
|
"https://api.stlouisfed.org/fred/series/observations", params=params
|
|
)
|
|
resp.raise_for_status()
|
|
payload = resp.json()
|
|
except Exception as exc:
|
|
logger.warning("Regime monitor: FRED fetch failed for %s: %s", series_id, exc)
|
|
return None
|
|
|
|
out: Series = []
|
|
for obs in payload.get("observations", []):
|
|
value = obs.get("value")
|
|
if value in (None, ".", ""):
|
|
continue
|
|
try:
|
|
out.append((date.fromisoformat(obs["date"]), float(value)))
|
|
except (TypeError, ValueError):
|
|
continue
|
|
return sorted(out, key=lambda x: x[0])
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Snapshot persistence
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def _upsert_snapshot(db: AsyncSession, result: dict) -> None:
|
|
d = date.fromisoformat(result["date"])
|
|
existing = await db.execute(select(RegimeSnapshot).where(RegimeSnapshot.date == d))
|
|
row = existing.scalar_one_or_none()
|
|
payload = json.dumps(result)
|
|
if row is None:
|
|
db.add(RegimeSnapshot(
|
|
date=d,
|
|
total_score=result["total_score"],
|
|
band=result["band"],
|
|
breakdown_json=payload,
|
|
created_at=datetime.now(timezone.utc),
|
|
))
|
|
else:
|
|
row.total_score = result["total_score"]
|
|
row.band = result["band"]
|
|
row.breakdown_json = payload
|
|
|
|
|
|
async def _snapshot_count(db: AsyncSession) -> int:
|
|
res = await db.execute(select(func.count()).select_from(RegimeSnapshot))
|
|
return int(res.scalar() or 0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Job entrypoint + reads
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def update_regime_monitor(db: AsyncSession, backfill_days: int = 90) -> dict:
|
|
"""Compute the latest index, persist it, and backfill history on first run.
|
|
|
|
Job entrypoint (daily-pipeline step). Best-effort throughout: missing keys or
|
|
a failed source degrade gracefully (signals drop to n/a) rather than abort.
|
|
"""
|
|
config = await get_regime_config(db)
|
|
|
|
# Refresh the LLM fundamentals if stale (and not manually locked). Best-effort.
|
|
overrides = await get_fundamental_overrides(db)
|
|
if _fundamentals_stale(overrides, config) and not overrides.get("locked"):
|
|
try:
|
|
overrides = await refresh_fundamental_overrides(db, config=config)
|
|
except Exception as exc:
|
|
logger.warning("Regime monitor: fundamentals refresh skipped: %s", exc)
|
|
|
|
end = date.today()
|
|
start = end - timedelta(days=400)
|
|
prices = await _fetch_prices(config, start, end)
|
|
vix_series = await _fetch_fred_series("VIXCLS", start, end)
|
|
oas_series = await _fetch_fred_series("BAMLH0A0HYM2", end - timedelta(days=1200), end)
|
|
|
|
# Anchor "today" on the latest actual trading day we have prices for.
|
|
leader = config["tickers"]["leaders"][0] if config["tickers"]["leaders"] else None
|
|
leader_series = prices.get(leader or "", [])
|
|
latest_date = leader_series[-1][0] if leader_series else end
|
|
|
|
# Early-warning signal: breadth-divergence over the stored universe (leads but
|
|
# noisy). Computed once here so the daily job carries it live, as a SEPARATE
|
|
# score next to the coincident index — not folded into the index weights.
|
|
# Best-effort: a breadth failure must not stop the index update.
|
|
try:
|
|
breadth = await breadth_service.compute_breadth_series(db)
|
|
divergence = breadth_service.compute_divergence_series(breadth, sorted(leader_series))
|
|
except Exception as exc:
|
|
logger.warning("Regime monitor: breadth/divergence skipped: %s", exc)
|
|
divergence = {}
|
|
# As-of lookup: the stored universe (breadth) can lag the live benchmark date
|
|
# by a day or two, so an exact-date match would blank the newest snapshot.
|
|
div_items = sorted(divergence.items())
|
|
cw = config.get("combined_weights") or {"coincident": 0.6, "early_warning": 0.4}
|
|
|
|
dates = {latest_date}
|
|
if await _snapshot_count(db) < 5 and leader_series:
|
|
cutoff = end - timedelta(days=backfill_days)
|
|
dates |= {d for d, _ in leader_series if d >= cutoff}
|
|
|
|
latest_result: dict | None = None
|
|
for d in sorted(dates):
|
|
result = _compute_index(prices, vix_series, oas_series, overrides, config, d)
|
|
_attach_early_warning(result, _divergence_asof(div_items, d), cw)
|
|
await _upsert_snapshot(db, result)
|
|
latest_result = result
|
|
|
|
# Backfill early-warning + combined onto recent existing snapshots (e.g. the
|
|
# index history written before this signal existed) so their 7/30-day trends
|
|
# populate immediately rather than only filling in over the coming weeks.
|
|
if div_items:
|
|
recent = await db.execute(
|
|
select(RegimeSnapshot).where(RegimeSnapshot.date >= end - timedelta(days=120))
|
|
)
|
|
for row in recent.scalars().all():
|
|
try:
|
|
res = json.loads(row.breakdown_json)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if (res.get("early_warning") or {}).get("score") is not None:
|
|
continue
|
|
_attach_early_warning(res, _divergence_asof(div_items, row.date), cw)
|
|
row.breakdown_json = json.dumps(res)
|
|
await db.commit()
|
|
|
|
logger.info(json.dumps({
|
|
"event": "regime_monitor_updated",
|
|
"date": latest_result["date"] if latest_result else None,
|
|
"score": latest_result["total_score"] if latest_result else None,
|
|
"snapshots_written": len(dates),
|
|
}))
|
|
return latest_result or {"available": False, "reason": "no data"}
|
|
|
|
|
|
def _divergence_asof(div_items: list[tuple[date, float]], as_of: date, max_lag_days: int = 7) -> float | None:
|
|
"""Latest divergence value on/before ``as_of``, tolerating a small data lag
|
|
between the live benchmark and the stored universe. None if too stale/absent."""
|
|
chosen: tuple[date, float] | None = None
|
|
for d, v in div_items:
|
|
if d <= as_of:
|
|
chosen = (d, v)
|
|
else:
|
|
break
|
|
if chosen is None or (as_of - chosen[0]).days > max_lag_days:
|
|
return None
|
|
return chosen[1]
|
|
|
|
|
|
def _attach_early_warning(result: dict, ew: float | None, weights: dict) -> None:
|
|
"""Attach the separate early-warning score and a combined blend to a snapshot.
|
|
|
|
``ew`` is the breadth-divergence value as-of this date (or None). The combined
|
|
score is a normalised weighted mean of the coincident index and the early
|
|
warning — observational, kept apart from the index itself.
|
|
"""
|
|
result["early_warning"] = {
|
|
"score": round(ew, 1) if ew is not None else None,
|
|
"band": band_for(ew) if ew is not None else None,
|
|
}
|
|
if ew is None:
|
|
combined = result["total_score"]
|
|
else:
|
|
wc = float(weights.get("coincident", 0.6))
|
|
we = float(weights.get("early_warning", 0.4))
|
|
wsum = (wc + we) or 1.0
|
|
combined = (result["total_score"] * wc + ew * we) / wsum
|
|
result["combined"] = {"score": round(combined, 1), "band": band_for(combined)}
|
|
|
|
|
|
async def _result_at_or_before(db: AsyncSession, target: date) -> dict | None:
|
|
"""Parsed snapshot result for the latest date on/before ``target``."""
|
|
res = await db.execute(
|
|
select(RegimeSnapshot.breakdown_json)
|
|
.where(RegimeSnapshot.date <= target)
|
|
.order_by(RegimeSnapshot.date.desc())
|
|
.limit(1)
|
|
)
|
|
raw = res.scalar_one_or_none()
|
|
if raw is None:
|
|
return None
|
|
try:
|
|
return json.loads(raw)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
|
|
def _delta(curr: float | None, prev: float | None) -> float | None:
|
|
return round(curr - prev, 1) if (curr is not None and prev is not None) else None
|
|
|
|
|
|
async def get_regime_monitor(db: AsyncSession) -> dict:
|
|
"""Latest snapshot + 7/30-day trend deltas for the index, early-warning, and
|
|
combined scores. Cheap (a few row reads)."""
|
|
res = await db.execute(
|
|
select(RegimeSnapshot).order_by(RegimeSnapshot.date.desc()).limit(1)
|
|
)
|
|
latest = res.scalar_one_or_none()
|
|
if latest is None:
|
|
return {"available": False, "reason": "not computed yet"}
|
|
|
|
try:
|
|
result = json.loads(latest.breakdown_json)
|
|
except (TypeError, ValueError):
|
|
result = {"date": latest.date.isoformat(), "total_score": latest.total_score,
|
|
"band": latest.band, "breakdown": []}
|
|
|
|
r7 = await _result_at_or_before(db, latest.date - timedelta(days=7))
|
|
r30 = await _result_at_or_before(db, latest.date - timedelta(days=30))
|
|
|
|
def _nested(r: dict | None, key: str) -> float | None:
|
|
return (r.get(key) or {}).get("score") if r else None
|
|
|
|
result["available"] = True
|
|
cur_total = result.get("total_score")
|
|
result["trend"] = {
|
|
"delta_7": _delta(cur_total, (r7 or {}).get("total_score")),
|
|
"delta_30": _delta(cur_total, (r30 or {}).get("total_score")),
|
|
}
|
|
for key in ("early_warning", "combined"):
|
|
block = result.get(key) or {"score": None, "band": None}
|
|
block["delta_7"] = _delta(block.get("score"), _nested(r7, key))
|
|
block["delta_30"] = _delta(block.get("score"), _nested(r30, key))
|
|
result[key] = block
|
|
return result
|
|
|
|
|
|
async def get_regime_history(db: AsyncSession, days: int = 400) -> list[dict]:
|
|
"""Daily history of the index, early-warning, and combined scores for the
|
|
score-over-time chart. One point per snapshot date, ascending."""
|
|
cutoff = date.today() - timedelta(days=days)
|
|
res = await db.execute(
|
|
select(RegimeSnapshot)
|
|
.where(RegimeSnapshot.date >= cutoff)
|
|
.order_by(RegimeSnapshot.date.asc())
|
|
)
|
|
out: list[dict] = []
|
|
for row in res.scalars().all():
|
|
try:
|
|
data = json.loads(row.breakdown_json)
|
|
except (TypeError, ValueError):
|
|
data = {}
|
|
out.append({
|
|
"date": row.date.isoformat(),
|
|
"index": row.total_score,
|
|
"early_warning": (data.get("early_warning") or {}).get("score"),
|
|
"combined": (data.get("combined") or {}).get("score"),
|
|
})
|
|
return out
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# F1/F3 via grounded LLM (reuses the configured sentiment provider)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
_CAPEX_PROMPT = """\
|
|
You are a markets analyst. Search the web for the MOST RECENT (last reported \
|
|
quarter) capital-expenditure (capex) guidance from these hyperscalers: {names}.
|
|
|
|
For each name, classify the direction of its forward capex/AI-infrastructure \
|
|
guidance vs. the prior quarter as exactly one of: "raising", "holding", "cutting".
|
|
|
|
Also judge the recent "good news, stock down" dynamic: across these names and \
|
|
the semiconductor sector, did stocks broadly FALL despite earnings/revenue beats \
|
|
in the last reporting season? Answer "yes", "no", or "mixed".
|
|
|
|
Respond ONLY with a JSON object (no markdown):
|
|
{{"capex": {{ {example} }}, "good_news_stock_down": "yes|no|mixed", \
|
|
"reasoning": "<2-3 sentences citing the specific guidance you found>"}}
|
|
"""
|
|
|
|
|
|
def _fundamentals_stale(overrides: dict, config: dict) -> bool:
|
|
fetched = overrides.get("fetched_at")
|
|
if not fetched:
|
|
return True
|
|
try:
|
|
ts = datetime.fromisoformat(fetched)
|
|
except (TypeError, ValueError):
|
|
return True
|
|
if ts.tzinfo is None:
|
|
ts = ts.replace(tzinfo=timezone.utc)
|
|
max_age = timedelta(days=int(config.get("fundamental_staleness_days", 80)))
|
|
return datetime.now(timezone.utc) - ts > max_age
|
|
|
|
|
|
def _strip_fences(text: str) -> str:
|
|
clean = (text or "").strip()
|
|
if clean.startswith("```"):
|
|
clean = clean.split("\n", 1)[1] if "\n" in clean else clean[3:]
|
|
if clean.endswith("```"):
|
|
clean = clean[:-3]
|
|
return clean.strip()
|
|
|
|
|
|
def _extract_responses_text(response: object) -> str:
|
|
for item in getattr(response, "output", []) or []:
|
|
if getattr(item, "type", None) == "message" and getattr(item, "content", None):
|
|
for block in item.content:
|
|
if getattr(block, "text", None):
|
|
return block.text
|
|
return ""
|
|
|
|
|
|
async def _call_llm_json(cfg: dict, prompt: str) -> dict:
|
|
"""Send one grounded prompt via the configured LLM and parse its JSON reply."""
|
|
provider, model, api_key = cfg["provider"], cfg["model"], cfg["api_key"]
|
|
base_url = cfg.get("base_url")
|
|
|
|
if provider == "gemini":
|
|
from google import genai
|
|
from google.genai import types
|
|
client = genai.Client(api_key=api_key)
|
|
resp = await client.aio.models.generate_content(
|
|
model=model,
|
|
contents=prompt,
|
|
config=types.GenerateContentConfig(
|
|
tools=[types.Tool(google_search=types.GoogleSearch())],
|
|
response_mime_type="application/json",
|
|
),
|
|
)
|
|
return json.loads(_strip_fences(resp.text))
|
|
|
|
from openai import AsyncOpenAI
|
|
verify = _CA_BUNDLE if (_CA_BUNDLE and Path(_CA_BUNDLE).exists()) else True
|
|
client = AsyncOpenAI(
|
|
api_key=api_key,
|
|
base_url=base_url or None,
|
|
http_client=httpx.AsyncClient(verify=verify),
|
|
)
|
|
if provider in ("openai", "xai"):
|
|
tool = "web_search_preview" if provider == "openai" else "web_search"
|
|
resp = await client.responses.create(
|
|
model=model,
|
|
tools=[{"type": tool}],
|
|
instructions="Respond with valid JSON only, no markdown fences.",
|
|
input=prompt,
|
|
)
|
|
return json.loads(_strip_fences(_extract_responses_text(resp)))
|
|
|
|
# deepseek / generic OpenAI-compatible: no web search, knowledge-based.
|
|
resp = await client.chat.completions.create(
|
|
model=model,
|
|
messages=[{"role": "user", "content": prompt}],
|
|
response_format={"type": "json_object"},
|
|
)
|
|
return json.loads(_strip_fences(resp.choices[0].message.content))
|
|
|
|
|
|
_CAPEX_STATE_SCORES = {"raising": 0.0, "holding": 50.0, "cutting": 100.0}
|
|
_GNSD_SCORES = {"yes": 100.0, "mixed": 50.0, "no": 0.0}
|
|
|
|
|
|
async def refresh_fundamental_overrides(
|
|
db: AsyncSession, config: dict | None = None, force: bool = False
|
|
) -> dict:
|
|
"""Ask the configured LLM to propose F1 (capex) and F3 (earnings reaction).
|
|
|
|
Skips (returns current) if a manual override is locked, unless ``force``.
|
|
"""
|
|
current = await get_fundamental_overrides(db)
|
|
if current.get("locked") and not force:
|
|
return current
|
|
|
|
config = config or await get_regime_config(db)
|
|
cfg = await resolve_llm_config(db)
|
|
if not cfg.get("api_key"):
|
|
raise ProviderError(f"No API key configured for LLM provider '{cfg.get('provider')}'")
|
|
|
|
names = config["tickers"]["hyperscalers"]
|
|
example = ", ".join(f'"{n}": "holding"' for n in names)
|
|
prompt = _CAPEX_PROMPT.format(names=", ".join(names), example=example)
|
|
parsed = await _call_llm_json(cfg, prompt)
|
|
|
|
capex = parsed.get("capex", {}) if isinstance(parsed, dict) else {}
|
|
scores = [
|
|
_CAPEX_STATE_SCORES[str(capex.get(n, "")).strip().lower()]
|
|
for n in names
|
|
if str(capex.get(n, "")).strip().lower() in _CAPEX_STATE_SCORES
|
|
]
|
|
f1 = _mean(scores) if scores else 50.0
|
|
gnsd = str(parsed.get("good_news_stock_down", "")).strip().lower()
|
|
f3 = _GNSD_SCORES.get(gnsd, 50.0)
|
|
|
|
result = {
|
|
"f1_score": round(f1, 1),
|
|
"f3_score": f3,
|
|
"capex": capex,
|
|
"good_news_stock_down": gnsd or None,
|
|
"reasoning": parsed.get("reasoning") if isinstance(parsed, dict) else None,
|
|
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
|
"locked": False,
|
|
"source": cfg.get("provider"),
|
|
}
|
|
await update_setting(db, KEY_FUNDAMENTALS, json.dumps(result))
|
|
logger.info(json.dumps({"event": "regime_fundamentals_refreshed", "f1": result["f1_score"], "f3": result["f3_score"]}))
|
|
return result
|