signal-platform/app/services/regime_monitor_service.py

"""AI/Tech Regime Monitor v3.

The monitor is a risk thermometer, not a probability or trading rule.  It keeps
two deliberately separate outputs:

* State: current structural stress (price, breadth, credit, volatility).
* Warning: deterioration/divergence that may precede State (breadth divergence,
  relative strength, credit impulse).

Both scores are quantitative and daily.  The sourced hyperscaler capex and
earnings-reaction observations are a qualitative *overlay* in v3 rather than
weighted sensors: at a combined 20 points they could not reach the event
study's alarm threshold even when both pegged, so refreshing them appeared to
do nothing.  They are reported next to the scores instead of inside them.

Daily snapshots are the point-in-time record.  The first run under a new
``METHODOLOGY`` rewrites the latest ``REBUILD_SESSIONS`` trading sessions once;
ordinary runs thereafter only upsert the latest trading date.  The overlay is
still gated by its effective date so a rebuild cannot stamp today's observation
onto historical snapshots.
"""

from __future__ import annotations

import hashlib
import json
import logging
import os
from datetime import date, datetime, timedelta, timezone
from pathlib import Path

import httpx
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from app.config import settings
from app.exceptions import ProviderError, ValidationError
from app.models.regime_snapshot import RegimeSnapshot
from app.providers.alpaca import AlpacaOHLCVProvider
from app.services import breadth_service, settings_store
from app.services.admin_service import update_setting
from app.services.sentiment_provider_service import _resolve as resolve_llm_config

logger = logging.getLogger(__name__)

_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")

KEY_CONFIG = "regime_monitor_config"
KEY_FUNDAMENTALS = "regime_fundamental_overrides"

METHODOLOGY = "v3"
# Snapshots are reseeded on a methodology bump, but fundamental observations are
# collected by hand/LLM and carried across it when the format is compatible.
CATEGORICAL_FUNDAMENTAL_METHODOLOGIES = frozenset({"v2", "v3"})
REBUILD_SESSIONS = 400
MIN_COVERAGE = 75.0
SOURCE_MAX_LAG_DAYS = 7

# Bands are per axis: the two scores have genuinely different realized ranges,
# so one shared set made Warning's top bands unreachable (v2 Warning never
# exceeded 64.9 in 408 sessions while State reached 91.2).  Thresholds are round
# numbers chosen so each band covers a sane share of history, not percentile
# fits -- percentile-derived bands would drift on every rebuild and silently
# rewrite what past snapshots meant.  Realized shares over the 408 sessions to
# 2026-07-24: State 73/15/8/3%, Warning 69/20/8/3%.
STATE_BANDS = (20.0, 50.0, 80.0)
WARNING_BANDS = (20.0, 40.0, 60.0)

QUADRANT_STATE_DIVIDER = 50.0
QUADRANT_WARNING_DIVIDER = 40.0
QUADRANT_MARGIN = 5.0

HY_OAS_MILD = 3.5
HY_OAS_ELEVATED = 5.0
HY_OAS_STRESSED = 7.0
# ICE restricted FRED to a rolling 3-year window for BAMLH0A0HYM2 in April 2026
# ("Starting in April 2026, this series will only include 3 years of
# observations"), so v2's 10-year reference window silently became 3.  Against 3
# years of uniformly tight spreads (2.59-4.61 over the calibration window) the
# blended upper-tail percentile saturated at an OAS of ~4.5 and scored 20 points
# of stress at 3.5 -- the level these anchors call "mild".  The anchors already
# encode the long-run distribution, so the credit *level* is now purely anchored
# and credit *dynamics* live in W3 on the Warning axis where they belong.
HY_OAS_WINDOW_DAYS = 400  # only W3's lookback plus slack is needed now
W3_OAS_LOOKBACK = 20
W3_OAS_FULL_SCALE_PCT = 35.0

# Drawdown anchors (drawdown %, stress score).  v2 used a bare ``dd_pct * 5``,
# which pegged at a 20% drawdown -- the 90th percentile of the observed
# distribution -- so 39 of 408 sessions sat at exactly 100 with no resolution
# left during the part of a selloff that matters most.  These anchors keep
# headroom past the observed 36% maximum.
P3_DRAWDOWN_ANCHORS = (
    (0.0, 0.0), (4.0, 10.0), (8.0, 25.0), (16.0, 50.0), (28.0, 78.0), (40.0, 100.0),
)

STATE_WEIGHTS = {
    "price": 40.0,
    "breadth": 25.0,
    "credit": 20.0,
    "volatility": 15.0,
}
# Fundamentals left the score in v3.  At 12 + 8 points they could not reach the
# event study's alarm threshold even when both pegged at 100, so the LLM read was
# decorative; it is now a separate qualitative overlay.  Credit *impulse* takes
# their place because the OAS level is pinned at zero below the 3.5 anchor while
# its rate of change is not.
WARNING_WEIGHTS = {
    "breadth_divergence": 45.0,
    "relative_strength": 30.0,
    "credit_impulse": 25.0,
}

# Fixed at the v2 launch.  These are liquid S&P 500/Nasdaq AI, semiconductor,
# infrastructure, cloud, and enterprise-software names that the platform's
# normal universe sync already stores.
DEFAULT_BREADTH_BASKET = [
    "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "AVGO", "AMD",
    "ORCL", "CRM", "NOW", "PLTR", "ANET", "DELL", "SMCI", "MU",
    "QCOM", "INTC", "AMAT", "LRCX", "KLAC", "SNPS", "CDNS", "ADI",
    "TXN", "IBM", "CSCO", "PANW", "CRWD", "VRT",
]

DEFAULT_CONFIG: dict = {
    "tickers": {
        "leaders": ["SMH"],
        "confirm": ["QQQ"],
        "market": "SPY",
        "hyperscalers": ["GOOGL", "AMZN", "META", "MSFT"],
    },
    "breadth_basket": DEFAULT_BREADTH_BASKET,
    "basket_asof": "2026-07-15",
    "fundamental_staleness_days": 80,
}

CAPEX_STATES = ("raising", "holding", "cutting", "unknown")
GNSD_STATES = ("yes", "no", "mixed")
# v2 scored raising and holding identically at 0, so in a capex boom the reading
# was pinned at 0 and could not express the raising -> holding deceleration that
# is the actual early warning. Display-only in v3, but it should still describe.
_CAPEX_STATE_SCORES = {"raising": 0.0, "holding": 50.0, "cutting": 100.0}
_GNSD_SCORES = {"yes": 100.0, "no": 0.0}

Series = list[tuple[date, float]]


# ---------------------------------------------------------------------------
# Pure numeric helpers and sensors
# ---------------------------------------------------------------------------

def _clamp(x: float, lo: float = 0.0, hi: float = 100.0) -> float:
    return max(lo, min(hi, x))


def _sma(values: list[float], window: int) -> float | None:
    if len(values) < window:
        return None
    return sum(values[-window:]) / window


def _mean(values: list[float]) -> float | None:
    return sum(values) / len(values) if values else None


def _blend(leader: float | None, confirm: float | None, leader_weight: float = 2.0) -> float | None:
    parts: list[tuple[float, float]] = []
    if leader is not None:
        parts.append((leader, leader_weight))
    if confirm is not None:
        parts.append((confirm, 1.0))
    if not parts:
        return None
    return sum(v * w for v, w in parts) / sum(w for _, w in parts)


def _interpolate(x: float, anchors: tuple[tuple[float, float], ...]) -> float:
    """Piecewise-linear lookup, flat outside the first and last anchor."""
    if x <= anchors[0][0]:
        return anchors[0][1]
    for (x0, y0), (x1, y1) in zip(anchors, anchors[1:]):
        if x <= x1:
            return y0 + (y1 - y0) * (x - x0) / (x1 - x0)
    return anchors[-1][1]


def band_for(score: float, bands: tuple[float, float, float] = STATE_BANDS) -> str:
    watch, elevated, breaking = bands
    if score < watch:
        return "stable"
    if score < elevated:
        return "watch"
    if score < breaking:
        return "elevated"
    return "breaking"


def _under_200(closes: list[float]) -> float | None:
    sma200 = _sma(closes, 200)
    if sma200 is None:
        return None
    return 100.0 if closes[-1] < sma200 else 0.0


def p1_trend_break(smh: list[float], qqq: list[float], leader_weight: float = 2.0) -> float | None:
    return _blend(_under_200(smh), _under_200(qqq), leader_weight)


def _death_cross(closes: list[float]) -> float | None:
    sma50 = _sma(closes, 50)
    sma200 = _sma(closes, 200)
    if sma50 is None or sma200 is None or len(closes) < 221 or sma200 == 0:
        return None
    gap_pct = (sma50 / sma200 - 1.0) * 100.0
    severity = 0.0 if gap_pct >= 0 else _clamp(-gap_pct * 20.0)
    sma200_past = _sma(closes[:-20], 200)
    if sma200_past:
        slope_pct = (sma200 / sma200_past - 1.0) * 100.0
        if slope_pct >= 0:
            severity *= 0.5
    return severity


def p2_death_cross(smh: list[float], qqq: list[float], leader_weight: float = 2.0) -> float | None:
    return _blend(_death_cross(smh), _death_cross(qqq), leader_weight)


def drawdown_pct(closes: list[float]) -> float | None:
    """Percentage below the trailing 52-week closing high."""
    if len(closes) < 30:
        return None
    peak = max(closes[-252:])
    if peak <= 0:
        return None
    return (peak - closes[-1]) / peak * 100.0


def _drawdown(closes: list[float]) -> float | None:
    dd_pct = drawdown_pct(closes)
    return None if dd_pct is None else _clamp(_interpolate(dd_pct, P3_DRAWDOWN_ANCHORS))


def p3_drawdown(smh: list[float], qqq: list[float], leader_weight: float = 2.0) -> float | None:
    """Anchored drawdown stress on the same 2:1 leader/confirm blend P1 and P2 use.

    v2 took ``max()`` here, which meant the more volatile leader always won and
    the price pillar reduced to this one sensor: its realized share of State was
    65% against a nominal 40% weight.  Blending brings that back to 40%.
    """
    return _blend(_drawdown(smh), _drawdown(qqq), leader_weight)


def p4_relative_strength(smh: list[float], spy: list[float], lookback: int = 60) -> float | None:
    """Stress-only SMH/SPY rollover: flat/outperformance=0, -10%=100."""
    if len(smh) < lookback + 1 or len(spy) < lookback + 1:
        return None
    if spy[-1] == 0 or spy[-lookback - 1] == 0:
        return None
    now = smh[-1] / spy[-1]
    past = smh[-lookback - 1] / spy[-lookback - 1]
    if past == 0:
        return None
    chg_pct = (now / past - 1.0) * 100.0
    return _clamp(-chg_pct * 10.0)


def p5_volatility(vix: float | None) -> float | None:
    if vix is None:
        return None
    return _clamp((vix - 15.0) / 15.0 * 100.0)


def breadth_level_score(pct_above_200: float | None) -> float | None:
    """Broad >=60%=healthy; <=20%=full breadth stress; linear between."""
    if pct_above_200 is None:
        return None
    return _clamp((60.0 - pct_above_200) / 40.0 * 100.0)


def _oas_absolute_score(value: float) -> float:
    if value <= HY_OAS_MILD:
        return 0.0
    if value <= HY_OAS_ELEVATED:
        return (value - HY_OAS_MILD) / (HY_OAS_ELEVATED - HY_OAS_MILD) * 50.0
    if value < HY_OAS_STRESSED:
        return 50.0 + (value - HY_OAS_ELEVATED) / (HY_OAS_STRESSED - HY_OAS_ELEVATED) * 50.0
    return 100.0


def f2_credit_spreads(oas_values: list[float]) -> float | None:
    """HY OAS level against named absolute anchors (3.5 mild / 5.0 / 7.0).

    v2 blended 70% of this with a 30% upper-tail percentile over the available
    history. That leg was always a second, noisier estimate of what the anchors
    already encode -- and once the usable window shrank to 3 uniformly tight
    years it saturated far below any real stress level. Removed rather than
    repaired: see ``HY_OAS_WINDOW_DAYS``.
    """
    if not oas_values:
        return None
    return round(_oas_absolute_score(oas_values[-1]), 2)


def w3_credit_impulse(
    oas_values: list[float], lookback: int = W3_OAS_LOOKBACK
) -> float | None:
    """HY OAS rate of change: widening only, relative so it works at any level.

    The credit *level* (C1) sits at zero for as long as spreads stay under the
    3.5 mild anchor -- 2.77 as of the v3 cutover -- so it contributes nothing to
    State in a calm tape.  The rate of change still does, and spread widening is
    a classic lead, which is what Warning is for.  Relative rather than absolute
    because +0.5pp means something very different at 2.7 than at 8.0.
    """
    if len(oas_values) < lookback + 1:
        return None
    past = oas_values[-lookback - 1]
    if past <= 0:
        return None
    change_pct = (oas_values[-1] / past - 1.0) * 100.0
    return _clamp(change_pct / W3_OAS_FULL_SCALE_PCT * 100.0)


def warning_sensor_scores(
    divergence: float | None,
    leader_closes: list[float],
    market_closes: list[float],
    oas_window: list[float],
) -> dict[str, float | None]:
    """The three Warning sensors, by pillar id.

    Single definition so the live monitor and the event study cannot drift apart
    -- in v2 the study re-derived the score from ``WARNING_WEIGHTS`` by hand and
    would have silently kept measuring the old construct through this change.
    """
    return {
        "breadth_divergence": divergence,
        "relative_strength": p4_relative_strength(leader_closes, market_closes),
        "credit_impulse": w3_credit_impulse(oas_window),
    }


def score_warning_sensors(sensors: dict[str, float | None]) -> float | None:
    """Weighted Warning score, renormalised over the sensors that are available."""
    live = [
        (float(score), float(WARNING_WEIGHTS[key]))
        for key, score in sensors.items()
        if score is not None and key in WARNING_WEIGHTS
    ]
    if not live:
        return None
    return sum(s * w for s, w in live) / sum(w for _, w in live)


def _sensor(sensor_id: str, label: str, score: float | None, **details: object) -> dict:
    return {
        "id": sensor_id,
        "label": label,
        "score": round(score, 1) if score is not None else None,
        "available": score is not None,
        "details": details,
    }


def _score_pillars(
    pillars: list[dict],
    weights: dict[str, float],
    bands: tuple[float, float, float] = STATE_BANDS,
) -> dict:
    expected = sum(max(0.0, float(w)) for w in weights.values())
    available_weight = sum(
        max(0.0, float(weights.get(p["id"], 0.0)))
        for p in pillars
        if p.get("score") is not None
    )
    coverage = available_weight / expected * 100.0 if expected else 0.0
    score = None
    if available_weight:
        score = sum(
            float(p["score"]) * float(weights.get(p["id"], 0.0))
            for p in pillars
            if p.get("score") is not None
        ) / available_weight

    rows: list[dict] = []
    for pillar in pillars:
        row = dict(pillar)
        weight = float(weights.get(row["id"], 0.0))
        row["weight"] = weight
        row["available"] = row.get("score") is not None
        row["contribution"] = (
            round(float(row["score"]) * weight / available_weight, 2)
            if row["available"] and available_weight
            else 0.0
        )
        rows.append(row)

    rounded = round(score, 1) if score is not None else None
    return {
        "score": rounded,
        "band": (
            band_for(rounded, bands)
            if rounded is not None and coverage >= MIN_COVERAGE
            else None
        ),
        "bands": {"watch": bands[0], "elevated": bands[1], "breaking": bands[2]},
        "coverage": round(coverage, 1),
        "minimum_coverage": MIN_COVERAGE,
        "available_pillars": [p["id"] for p in rows if p["available"]],
        "pillars": rows,
    }


# ---------------------------------------------------------------------------
# Point-in-time helpers
# ---------------------------------------------------------------------------

def _closes_asof(series: Series, as_of: date) -> list[float]:
    return [v for d, v in series if d <= as_of]


def _item_asof(series: Series | None, as_of: date) -> tuple[date, float] | None:
    if not series:
        return None
    chosen: tuple[date, float] | None = None
    for item in series:
        if item[0] <= as_of:
            chosen = item
        else:
            break
    return chosen


def _value_asof(series: Series | None, as_of: date) -> float | None:
    item = _item_asof(series, as_of)
    return item[1] if item else None


def _window_asof(series: Series | None, as_of: date, days: int) -> list[float]:
    if not series:
        return []
    start = as_of - timedelta(days=days)
    return [v for d, v in series if start <= d <= as_of]


def _coverage_days(series: Series | None, as_of: date) -> int | None:
    """Span of history actually available at ``as_of``.

    Recorded in every snapshot because the v2 credit percentile degraded from a
    10-year to a 3-year reference silently when the upstream licence changed --
    nothing asserted the window it claimed, so nothing noticed for months.
    """
    dates = [d for d, _ in series or [] if d <= as_of]
    return (as_of - dates[0]).days if dates else None


def _next_weekday(d: date) -> date:
    candidate = d + timedelta(days=1)
    while candidate.weekday() >= 5:
        candidate += timedelta(days=1)
    return candidate


def _parse_date(value: object) -> date | None:
    if not value:
        return None
    try:
        return date.fromisoformat(str(value)[:10])
    except ValueError:
        return None


def _fundamental_effective_date(overrides: dict) -> date | None:
    explicit = _parse_date(overrides.get("effective_date"))
    if explicit:
        return explicit
    fetched = _parse_date(overrides.get("fetched_at"))
    return _next_weekday(fetched) if fetched else None


def fundamental_overlay(overrides: dict, config: dict, as_of: date) -> dict:
    """Point-in-time qualitative overlay. Never feeds State or Warning in v3.

    The effective-date gate stays even though nothing is scored from this: the
    400-session rebuild replays historical dates, and stamping today's LLM read
    onto 2024 snapshots would be plain lookahead in the stored record.
    """
    effective = _fundamental_effective_date(overrides)
    pending = effective is None or as_of < effective
    age = None if pending else (as_of - effective).days
    stale = bool(age is not None and age > int(config.get("fundamental_staleness_days", 80)))
    return {
        "available": not pending and not stale,
        "pending": pending,
        "stale": stale,
        "effective_date": effective.isoformat() if effective else None,
        "age_days": age,
        "capex": None if pending else overrides.get("capex"),
        "good_news_stock_down": None if pending else overrides.get("good_news_stock_down"),
        "capex_stress": None if pending else overrides.get("f1_score"),
        "earnings_stress": None if pending else overrides.get("f3_score"),
        "reasoning": None if pending else overrides.get("reasoning"),
        "source": overrides.get("source"),
        "fetched_at": overrides.get("fetched_at"),
    }


def _basket_hash(symbols: list[str]) -> str:
    canonical = ",".join(sorted({s.strip().upper() for s in symbols if s.strip()}))
    return hashlib.sha256(canonical.encode("utf-8")).hexdigest()[:12]


def _mapping_series(values: dict[date, float]) -> Series:
    return sorted(values.items(), key=lambda item: item[0])


def _compute_index(
    prices: dict[str, Series],
    vix_series: Series | None,
    oas_series: Series | None,
    overrides: dict,
    config: dict,
    as_of: date,
    breadth_series: Series | None = None,
    divergence_series: Series | None = None,
    breadth_counts: dict[date, int] | None = None,
) -> dict:
    """Compute the complete v2 State/Warning snapshot as of one trading date."""
    tickers = config["tickers"]
    smh = _closes_asof(prices.get(tickers["leaders"][0], []), as_of)
    qqq = _closes_asof(prices.get(tickers["confirm"][0], []), as_of)
    spy = _closes_asof(prices.get(tickers["market"], []), as_of)

    p1 = p1_trend_break(smh, qqq)
    p2 = p2_death_cross(smh, qqq)
    p3 = p3_drawdown(smh, qqq)
    price_values = [v for v in (p1, p2, p3) if v is not None]
    price_score = max(price_values) if price_values else None

    breadth_item = _item_asof(breadth_series, as_of)
    breadth_pct = breadth_item[1] if breadth_item else None
    breadth_score = breadth_level_score(breadth_pct)
    vix_item = _item_asof(vix_series, as_of)
    vix_score = p5_volatility(vix_item[1] if vix_item else None)
    oas_item = _item_asof(oas_series, as_of)
    oas_window = _window_asof(oas_series, as_of, HY_OAS_WINDOW_DAYS)
    credit_score = f2_credit_spreads(oas_window)

    divergence = _value_asof(divergence_series, as_of)
    sensors = warning_sensor_scores(divergence, smh, spy, oas_window)
    relative_strength = sensors["relative_strength"]
    credit_impulse = sensors["credit_impulse"]
    overlay = fundamental_overlay(overrides, config, as_of)

    state_pillars = [
        {
            "id": "price",
            "label": "Price structure",
            "score": round(price_score, 1) if price_score is not None else None,
            "sensors": [
                _sensor("P1", "Trend break (200-DMA)", p1),
                _sensor("P2", "Death cross + slope", p2),
                _sensor("P3", "Drawdown from 52w high", p3),
            ],
        },
        {
            "id": "breadth",
            "label": "Breadth level",
            "score": round(breadth_score, 1) if breadth_score is not None else None,
            "sensors": [_sensor("B1", "% basket above 200-DMA", breadth_score, pct_above_200=breadth_pct)],
        },
        {
            "id": "credit",
            "label": "Credit level",
            "score": round(credit_score, 1) if credit_score is not None else None,
            "sensors": [_sensor("C1", "HY option-adjusted spread", credit_score, oas=oas_item[1] if oas_item else None)],
        },
        {
            "id": "volatility",
            "label": "Volatility level",
            "score": round(vix_score, 1) if vix_score is not None else None,
            "sensors": [_sensor("V1", "VIX level", vix_score, vix=vix_item[1] if vix_item else None)],
        },
    ]
    warning_pillars = [
        {
            "id": "breadth_divergence",
            "label": "Breadth divergence",
            "score": round(divergence, 1) if divergence is not None else None,
            "sensors": [_sensor("W1", "Price holding while breadth narrows", divergence)],
        },
        {
            "id": "relative_strength",
            "label": "SMH/SPY rollover",
            "score": round(relative_strength, 1) if relative_strength is not None else None,
            "sensors": [_sensor("W2", "60-session relative-strength deterioration", relative_strength)],
        },
        {
            "id": "credit_impulse",
            "label": "Credit impulse",
            "score": round(credit_impulse, 1) if credit_impulse is not None else None,
            "sensors": [
                _sensor(
                    "W3",
                    f"HY OAS {W3_OAS_LOOKBACK}-session widening",
                    credit_impulse,
                    oas=oas_item[1] if oas_item else None,
                )
            ],
        },
    ]

    state = _score_pillars(state_pillars, STATE_WEIGHTS, STATE_BANDS)
    warning = _score_pillars(warning_pillars, WARNING_WEIGHTS, WARNING_BANDS)

    price_item = _item_asof(prices.get(tickers["leaders"][0]), as_of)
    dated_sources = {
        "price": price_item[0] if price_item else None,
        "breadth": breadth_item[0] if breadth_item else None,
        "vix": vix_item[0] if vix_item else None,
        "credit": oas_item[0] if oas_item else None,
    }
    source_ages = {
        key: (as_of - d).days for key, d in dated_sources.items() if d is not None
    }
    stale_inputs = [key for key, age in source_ages.items() if age > SOURCE_MAX_LAG_DAYS]
    basket = list(config["breadth_basket"])
    basket_count = None
    if breadth_counts and breadth_item:
        basket_count = breadth_counts.get(breadth_item[0])

    return {
        "methodology": METHODOLOGY,
        "date": as_of.isoformat(),
        "state": state,
        "warning": warning,
        "fundamental_overlay": overlay,
        "quadrant_config": {
            "state_divider": QUADRANT_STATE_DIVIDER,
            "warning_divider": QUADRANT_WARNING_DIVIDER,
            "margin": QUADRANT_MARGIN,
        },
        "basket": {
            "symbols": basket,
            "hash": _basket_hash(basket),
            "basket_asof": config["basket_asof"],
            "members_available": basket_count,
            "members_expected": len(basket),
            "history_kind": "forward" if as_of >= date.fromisoformat(config["basket_asof"]) else "retrospective",
        },
        "inputs": {
            "vix": round(vix_item[1], 2) if vix_item else None,
            "vix_date": vix_item[0].isoformat() if vix_item else None,
            "hy_oas": round(oas_item[1], 2) if oas_item else None,
            "hy_oas_date": oas_item[0].isoformat() if oas_item else None,
            "breadth_pct_above_200": round(breadth_pct, 1) if breadth_pct is not None else None,
            "breadth_date": breadth_item[0].isoformat() if breadth_item else None,
            "fundamentals_fetched_at": overrides.get("fetched_at"),
            "fundamentals_effective_date": overlay.get("effective_date"),
            "fundamentals_age_days": overlay.get("age_days"),
        },
        "data_quality": {
            "minimum_coverage": MIN_COVERAGE,
            "oldest_market_input_age_days": max(source_ages.values()) if source_ages else None,
            "stale_inputs": stale_inputs,
            "inputs_fresh": not stale_inputs,
            # Upstream history spans, so a provider silently truncating a series
            # shows up in the record instead of quietly reshaping a sensor.
            "credit_history_days": _coverage_days(oas_series, as_of),
            "vix_history_days": _coverage_days(vix_series, as_of),
        },
    }


# ---------------------------------------------------------------------------
# Configuration and fundamental storage
# ---------------------------------------------------------------------------

def _normalise_basket(symbols: list[str]) -> list[str]:
    cleaned = [str(s).strip().upper().replace(".", "-") for s in symbols if str(s).strip()]
    if len(cleaned) != len(set(cleaned)):
        raise ValidationError("Breadth basket symbols must be unique")
    if not 20 <= len(cleaned) <= 100:
        raise ValidationError("Breadth basket must contain between 20 and 100 symbols")
    return cleaned


async def get_regime_config(db: AsyncSession) -> dict:
    cfg = json.loads(json.dumps(DEFAULT_CONFIG))
    raw = await settings_store.get_value(db, KEY_CONFIG)
    if raw:
        try:
            stored = json.loads(raw)
            if isinstance(stored.get("breadth_basket"), list):
                cfg["breadth_basket"] = _normalise_basket(stored["breadth_basket"])
            if stored.get("basket_asof"):
                cfg["basket_asof"] = str(stored["basket_asof"])
            if stored.get("fundamental_staleness_days") is not None:
                cfg["fundamental_staleness_days"] = int(stored["fundamental_staleness_days"])
        except (TypeError, ValueError, ValidationError):
            logger.warning("Corrupt %s; using v2 defaults", KEY_CONFIG)
    return cfg


async def update_regime_config(db: AsyncSession, updates: dict) -> dict:
    cfg = await get_regime_config(db)
    if "breadth_basket" in updates:
        basket = _normalise_basket(updates["breadth_basket"])
        if basket != cfg["breadth_basket"]:
            cfg["breadth_basket"] = basket
            cfg["basket_asof"] = date.today().isoformat()
    if "fundamental_staleness_days" in updates:
        days = int(updates["fundamental_staleness_days"])
        if not 30 <= days <= 180:
            raise ValidationError("Fundamental staleness must be between 30 and 180 days")
        cfg["fundamental_staleness_days"] = days
    await update_setting(db, KEY_CONFIG, json.dumps(cfg))
    return cfg


async def get_fundamental_overrides(db: AsyncSession) -> dict:
    names = DEFAULT_CONFIG["tickers"]["hyperscalers"]
    default = {
        "methodology": METHODOLOGY,
        "f1_score": None,
        "f3_score": None,
        "capex": {name: "unknown" for name in names},
        "good_news_stock_down": "mixed",
        "locked": False,
        "reasoning": None,
        "fetched_at": None,
        "effective_date": None,
        "source": "default",
    }
    raw = await settings_store.get_value(db, KEY_FUNDAMENTALS)
    if not raw:
        return default
    try:
        stored = json.loads(raw)
    except (TypeError, ValueError):
        return default
    # The guard rejects pre-v2 blobs, where f1/f3 were arbitrary numbers with no
    # categorical source. v2 and v3 share the categorical format and both derive
    # f1/f3 from it below, so a methodology bump must not discard a live
    # observation -- only the capex *scale* changed, and that is recomputed.
    if stored.get("methodology") not in CATEGORICAL_FUNDAMENTAL_METHODOLOGIES:
        return default
    capex = _normalise_capex_states(stored.get("capex"), names)
    reaction = str(stored.get("good_news_stock_down", "mixed")).strip().lower()
    if reaction not in GNSD_STATES:
        reaction = "mixed"
    return {
        **default,
        **stored,
        "methodology": METHODOLOGY,
        "f1_score": _score_capex_states(capex, names),
        "f3_score": _GNSD_SCORES.get(reaction),
        "capex": capex,
        "good_news_stock_down": reaction,
    }


def _normalise_capex_states(
    raw: object,
    names: list[str],
    *,
    strict: bool = False,
) -> dict[str, str]:
    values = raw if isinstance(raw, dict) else {}
    if strict and set(values) != set(names):
        raise ValidationError(
            f"Capex override must contain exactly: {', '.join(names)}"
        )
    out: dict[str, str] = {}
    for name in names:
        state = str(values.get(name, "unknown")).strip().lower()
        if state not in CAPEX_STATES:
            if strict:
                raise ValidationError(f"Invalid capex state for {name}: {state}")
            state = "unknown"
        out[name] = state
    return out


def _score_capex_states(capex: dict[str, str], names: list[str]) -> float | None:
    scores = [_CAPEX_STATE_SCORES[capex[name]] for name in names if capex[name] in _CAPEX_STATE_SCORES]
    score = _mean(scores) if len(scores) >= 3 else None
    return round(score, 1) if score is not None else None


async def set_fundamental_overrides(
    db: AsyncSession,
    capex: dict[str, str] | None = None,
    good_news_stock_down: str | None = None,
    locked: bool | None = None,
) -> dict:
    current = await get_fundamental_overrides(db)
    observation_changed = capex is not None or good_news_stock_down is not None
    if capex is not None:
        names = DEFAULT_CONFIG["tickers"]["hyperscalers"]
        normalised = _normalise_capex_states(capex, names, strict=True)
        current["capex"] = normalised
        current["f1_score"] = _score_capex_states(normalised, names)
    if good_news_stock_down is not None:
        reaction = good_news_stock_down.strip().lower()
        if reaction not in GNSD_STATES:
            raise ValidationError(f"Invalid good-news-stock-down state: {reaction}")
        current["good_news_stock_down"] = reaction
        current["f3_score"] = _GNSD_SCORES.get(reaction)
    if locked is not None:
        current["locked"] = bool(locked)
    elif observation_changed:
        current["locked"] = True
    if observation_changed:
        now = datetime.now(timezone.utc)
        current.update({
            "methodology": METHODOLOGY,
            "source": "manual",
            "reasoning": None,
            "fetched_at": now.isoformat(),
            "effective_date": _next_weekday(now.date()).isoformat(),
        })
    await update_setting(db, KEY_FUNDAMENTALS, json.dumps(current))
    return current


# ---------------------------------------------------------------------------
# External data fetching
# ---------------------------------------------------------------------------

def _price_symbols(config: dict) -> list[str]:
    tickers = config["tickers"]
    symbols = list(tickers["leaders"]) + list(tickers["confirm"]) + [tickers["market"]]
    return list(dict.fromkeys(s for s in symbols if s))


async def _fetch_prices(config: dict, start: date, end: date) -> dict[str, Series]:
    if not settings.alpaca_api_key or not settings.alpaca_api_secret:
        return {}
    provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
    out: dict[str, Series] = {}
    for symbol in _price_symbols(config):
        try:
            bars = await provider.fetch_ohlcv(symbol, start, end)
            out[symbol] = sorted(((b.date, float(b.close)) for b in bars), key=lambda item: item[0])
        except Exception as exc:
            logger.warning("Regime monitor: price fetch failed for %s: %s", symbol, exc)
    return out


async def _fetch_fred_series(series_id: str, start: date, end: date) -> Series | None:
    if not settings.fred_api_key:
        return None
    verify = _CA_BUNDLE if (_CA_BUNDLE and Path(_CA_BUNDLE).exists()) else True
    params = {
        "series_id": series_id,
        "api_key": settings.fred_api_key,
        "file_type": "json",
        "observation_start": start.isoformat(),
        "observation_end": end.isoformat(),
    }
    try:
        async with httpx.AsyncClient(timeout=30, verify=verify) as client:
            response = await client.get(
                "https://api.stlouisfed.org/fred/series/observations", params=params
            )
            response.raise_for_status()
            payload = response.json()
    except Exception as exc:
        logger.warning("Regime monitor: FRED fetch failed for %s: %s", series_id, exc)
        return None

    out: Series = []
    for observation in payload.get("observations", []):
        value = observation.get("value")
        if value in (None, ".", ""):
            continue
        try:
            out.append((date.fromisoformat(observation["date"]), float(value)))
        except (TypeError, ValueError):
            continue
    return sorted(out, key=lambda item: item[0])


# ---------------------------------------------------------------------------
# Snapshot persistence and reads
# ---------------------------------------------------------------------------

async def _upsert_snapshot(
    db: AsyncSession,
    result: dict,
    *,
    rewrite_existing_v2: bool,
) -> tuple[bool, dict]:
    snapshot_date = date.fromisoformat(result["date"])
    existing = await db.execute(select(RegimeSnapshot).where(RegimeSnapshot.date == snapshot_date))
    row = existing.scalar_one_or_none()
    state_score = (result.get("state") or {}).get("score")
    state_band = (result.get("state") or {}).get("band")
    payload = json.dumps(result)
    if row is None:
        db.add(RegimeSnapshot(
            date=snapshot_date,
            total_score=float(state_score or 0.0),
            band=state_band or "unavailable",
            breakdown_json=payload,
            created_at=datetime.now(timezone.utc),
        ))
    else:
        existing_v2 = _parse_snapshot(row.breakdown_json)
        if existing_v2 is not None and not rewrite_existing_v2:
            return False, existing_v2
        row.total_score = float(state_score or 0.0)
        row.band = state_band or "unavailable"
        row.breakdown_json = payload
    return True, result


def _parse_snapshot(raw: str) -> dict | None:
    try:
        parsed = json.loads(raw)
    except (TypeError, ValueError):
        return None
    return parsed if parsed.get("methodology") == METHODOLOGY else None


async def _latest_snapshot_row(db: AsyncSession) -> tuple[RegimeSnapshot, dict] | None:
    result = await db.execute(
        select(RegimeSnapshot).order_by(RegimeSnapshot.date.desc()).limit(1000)
    )
    for row in result.scalars().all():
        parsed = _parse_snapshot(row.breakdown_json)
        if parsed is not None:
            return row, parsed
    return None


async def update_regime_monitor(db: AsyncSession, rebuild_sessions: int = REBUILD_SESSIONS) -> dict:
    config = await get_regime_config(db)
    overrides = await get_fundamental_overrides(db)
    if _fundamentals_stale(overrides, config) and not overrides.get("locked"):
        try:
            overrides = await refresh_fundamental_overrides(db, config=config)
        except Exception as exc:
            logger.warning("Regime monitor: fundamentals refresh skipped: %s", exc)

    end = date.today()
    prices = await _fetch_prices(config, end - timedelta(days=1200), end)
    leader = config["tickers"]["leaders"][0]
    leader_series = prices.get(leader, [])
    if not leader_series:
        return {"available": False, "reason": "no benchmark price data"}
    latest_date = leader_series[-1][0]

    vix_series = await _fetch_fred_series("VIXCLS", end - timedelta(days=1200), end)
    # Asking for 13 years was misleading once the licence capped the series at 3;
    # the level needs the latest point and W3 needs its lookback, nothing more.
    oas_series = await _fetch_fred_series(
        "BAMLH0A0HYM2", end - timedelta(days=HY_OAS_WINDOW_DAYS), end
    )

    basket = config["breadth_basket"]
    try:
        breadth, breadth_counts = await breadth_service.compute_breadth_details(
            db, basket, window=200, min_tickers=20
        )
        divergence = breadth_service.compute_divergence_series(breadth, leader_series)
    except Exception as exc:
        logger.warning("Regime monitor: fixed-basket breadth skipped: %s", exc)
        breadth, breadth_counts, divergence = {}, {}, {}

    latest_v2 = await _latest_snapshot_row(db)
    rebuilding = latest_v2 is None and bool(leader_series)
    if rebuilding:
        dates = [d for d, _ in leader_series[-max(1, rebuild_sessions):]]
    else:
        # Routine PIT rule: only the latest trading date may be inserted/updated.
        dates = [latest_date]

    breadth_series = _mapping_series(breadth)
    divergence_series = _mapping_series(divergence)
    latest_result: dict | None = None
    snapshots_written = 0
    for snapshot_date in dates:
        computed = _compute_index(
            prices,
            vix_series,
            oas_series,
            overrides,
            config,
            snapshot_date,
            breadth_series,
            divergence_series,
            breadth_counts,
        )
        written, latest_result = await _upsert_snapshot(
            db,
            computed,
            rewrite_existing_v2=rebuilding or snapshot_date == latest_date,
        )
        snapshots_written += int(written)
    await db.commit()

    logger.info(json.dumps({
        "event": "regime_monitor_updated",
        "methodology": METHODOLOGY,
        "date": latest_result.get("date") if latest_result else None,
        "state": ((latest_result or {}).get("state") or {}).get("score"),
        "warning": ((latest_result or {}).get("warning") or {}).get("score"),
        "snapshots_written": snapshots_written,
    }))
    return latest_result or {"available": False, "reason": "no data"}


async def _result_at_or_before(
    db: AsyncSession,
    target: date,
    basket_hash: str | None = None,
) -> dict | None:
    result = await db.execute(
        select(RegimeSnapshot.breakdown_json)
        .where(RegimeSnapshot.date <= target)
        .order_by(RegimeSnapshot.date.desc())
        .limit(1000)
    )
    for raw in result.scalars().all():
        parsed = _parse_snapshot(raw)
        parsed_hash = ((parsed or {}).get("basket") or {}).get("hash")
        if parsed is not None and (basket_hash is None or parsed_hash == basket_hash):
            return parsed
    return None


def _delta(current: dict, previous: dict | None) -> float | None:
    if not previous:
        return None
    if current.get("available_pillars") != previous.get("available_pillars"):
        return None
    a, b = current.get("score"), previous.get("score")
    return round(a - b, 1) if a is not None and b is not None else None


async def get_regime_monitor(db: AsyncSession) -> dict:
    latest = await _latest_snapshot_row(db)
    if latest is None:
        return {"available": False, "reason": "v2 not computed yet"}
    row, result = latest
    basket_hash = (result.get("basket") or {}).get("hash")
    previous_7 = await _result_at_or_before(
        db, row.date - timedelta(days=7), basket_hash
    )
    previous_30 = await _result_at_or_before(
        db, row.date - timedelta(days=30), basket_hash
    )

    for key in ("state", "warning"):
        block = result.get(key) or {}
        block["trend"] = {
            "delta_7": _delta(block, (previous_7 or {}).get(key)),
            "delta_30": _delta(block, (previous_30 or {}).get(key)),
        }
        result[key] = block

    snapshot_age = (date.today() - row.date).days
    quality = result.get("data_quality") or {}
    quality["snapshot_age_days"] = snapshot_age
    quality["is_fresh"] = bool(quality.get("inputs_fresh")) and snapshot_age <= 4
    result["data_quality"] = quality

    # The snapshot's overlay is the point-in-time record; the reader also wants
    # the current observation even when it is not effective until the next
    # session, because otherwise refreshing it looks like it did nothing.
    config = await get_regime_config(db)
    overrides = await get_fundamental_overrides(db)
    live = fundamental_overlay(overrides, config, date.today())
    live["observed_in_snapshot"] = bool((result.get("fundamental_overlay") or {}).get("available"))
    result["fundamental_context"] = live
    result["available"] = True
    return result


async def get_regime_history(db: AsyncSession, days: int = 800) -> list[dict]:
    cutoff = date.today() - timedelta(days=days)
    result = await db.execute(
        select(RegimeSnapshot)
        .where(RegimeSnapshot.date >= cutoff)
        .order_by(RegimeSnapshot.date.asc())
    )
    out: list[dict] = []
    for row in result.scalars().all():
        data = _parse_snapshot(row.breakdown_json)
        if data is None:
            continue
        state, warning = data.get("state") or {}, data.get("warning") or {}
        out.append({
            "date": row.date.isoformat(),
            "state": state.get("score") if state.get("band") is not None else None,
            "warning": warning.get("score") if warning.get("band") is not None else None,
            "state_coverage": state.get("coverage"),
            "warning_coverage": warning.get("coverage"),
            "basket_hash": (data.get("basket") or {}).get("hash"),
        })
    if not out:
        return out
    latest_hash = out[-1]["basket_hash"]
    if latest_hash is None:
        return out
    return [point for point in out if point["basket_hash"] == latest_hash]


# ---------------------------------------------------------------------------
# Grounded fundamental extraction
# ---------------------------------------------------------------------------

_CAPEX_PROMPT = """\
You are a markets analyst. Search the web for the MOST RECENT (last reported \
quarter) capital-expenditure (capex) guidance from these hyperscalers: {names}.

For each name, classify forward capex/AI-infrastructure guidance vs. the prior \
quarter as exactly one of: "raising", "holding", "cutting", "unknown".

Also judge the recent good-news-stock-down dynamic across these names and the \
semiconductor sector after earnings/revenue beats. Answer "yes", "no", or "mixed".

Respond ONLY with JSON (no markdown):
{{"capex": {{ {example} }}, "good_news_stock_down": "yes|no|mixed", \
"reasoning": "<2-3 sourced sentences>"}}
"""


def _fundamentals_stale(overrides: dict, config: dict) -> bool:
    fetched = overrides.get("fetched_at")
    if not fetched:
        return True
    try:
        timestamp = datetime.fromisoformat(fetched)
    except (TypeError, ValueError):
        return True
    if timestamp.tzinfo is None:
        timestamp = timestamp.replace(tzinfo=timezone.utc)
    return datetime.now(timezone.utc) - timestamp > timedelta(
        days=int(config.get("fundamental_staleness_days", 80))
    )


def _strip_fences(text: str) -> str:
    clean = (text or "").strip()
    if clean.startswith("```"):
        clean = clean.split("\n", 1)[1] if "\n" in clean else clean[3:]
    if clean.endswith("```"):
        clean = clean[:-3]
    return clean.strip()


def _extract_responses_text(response: object) -> str:
    for item in getattr(response, "output", []) or []:
        if getattr(item, "type", None) == "message" and getattr(item, "content", None):
            for block in item.content:
                if getattr(block, "text", None):
                    return block.text
    return ""


async def _call_llm_json(cfg: dict, prompt: str) -> dict:
    provider, model, api_key = cfg["provider"], cfg["model"], cfg["api_key"]
    base_url = cfg.get("base_url")
    if provider == "gemini":
        from google import genai
        from google.genai import types

        client = genai.Client(api_key=api_key)
        response = await client.aio.models.generate_content(
            model=model,
            contents=prompt,
            config=types.GenerateContentConfig(
                tools=[types.Tool(google_search=types.GoogleSearch())],
                response_mime_type="application/json",
            ),
        )
        return json.loads(_strip_fences(response.text))

    from openai import AsyncOpenAI

    verify = _CA_BUNDLE if (_CA_BUNDLE and Path(_CA_BUNDLE).exists()) else True
    client = AsyncOpenAI(
        api_key=api_key,
        base_url=base_url or None,
        http_client=httpx.AsyncClient(verify=verify),
    )
    if provider in ("openai", "xai"):
        tool = "web_search_preview" if provider == "openai" else "web_search"
        response = await client.responses.create(
            model=model,
            tools=[{"type": tool}],
            instructions="Respond with valid JSON only, no markdown fences.",
            input=prompt,
        )
        return json.loads(_strip_fences(_extract_responses_text(response)))

    response = await client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        response_format={"type": "json_object"},
    )
    return json.loads(_strip_fences(response.choices[0].message.content))


async def refresh_fundamental_overrides(
    db: AsyncSession, config: dict | None = None, force: bool = False
) -> dict:
    current = await get_fundamental_overrides(db)
    if current.get("locked") and not force:
        return current

    config = config or await get_regime_config(db)
    llm = await resolve_llm_config(db)
    if not llm.get("api_key"):
        raise ProviderError(f"No API key configured for LLM provider '{llm.get('provider')}'")

    names = config["tickers"]["hyperscalers"]
    example = ", ".join(f'"{name}": "holding"' for name in names)
    parsed = await _call_llm_json(
        llm, _CAPEX_PROMPT.format(names=", ".join(names), example=example)
    )
    raw_capex = parsed.get("capex", {}) if isinstance(parsed, dict) else {}
    capex = _normalise_capex_states(raw_capex, names)
    f1 = _score_capex_states(capex, names)
    reaction = str(parsed.get("good_news_stock_down", "")).strip().lower()
    if reaction not in GNSD_STATES:
        reaction = "mixed"
    f3 = _GNSD_SCORES.get(reaction)
    now = datetime.now(timezone.utc)
    result = {
        "methodology": METHODOLOGY,
        "f1_score": f1,
        "f3_score": f3,
        "capex": capex,
        "good_news_stock_down": reaction or None,
        "reasoning": parsed.get("reasoning") if isinstance(parsed, dict) else None,
        "fetched_at": now.isoformat(),
        "effective_date": _next_weekday(now.date()).isoformat(),
        "locked": False,
        "source": llm.get("provider"),
    }
    await update_setting(db, KEY_FUNDAMENTALS, json.dumps(result))
    logger.info(json.dumps({
        "event": "regime_fundamentals_refreshed",
        "f1": result["f1_score"],
        "f3": result["f3_score"],
        "effective_date": result["effective_date"],
    }))
    return result