"""AI/Tech Regime-Change Monitor.

A standalone, observational tool: it scores how far the AI/Tech bull regime has
deteriorated toward a re-rating, as a single 0-100 **index** (not a calibrated
probability), broken down per signal. It is intentionally decoupled — nothing
here feeds gates, scoring, alerts, or trade logic. It only computes a number for
its own tab.

Design mirrors ``market_regime_service``: benchmark/sector bars are pulled
directly via Alpaca (no Universe membership needed), macro inputs (VIX, HY credit
spreads) come from FRED, and the daily result is persisted as one
``RegimeSnapshot`` row per date so the UI can show a 7/30-day trend. On the first
run the history is backfilled by replaying the (already-fetched) price/FRED series
as-of each past day, so the trend is populated immediately.

Signals (sub-score 0 = healthy … 100 = regime breaking):
  P1 trend break (% under 200-DMA, SMH-led)     P2 death cross + 200-slope
  P3 drawdown from 52w high                       P4 relative strength SMH/SPY
  P5 volatility (VIX)                             P6 NVDA canary divergence (opt.)
  F1 hyperscaler capex guidance (LLM/manual)      F2 HY credit-spread percentile
  F3 "good news, stock down" (LLM/manual)         F4 market breadth RSP/SPY
"""

from __future__ import annotations

import copy
import json
import logging
import os
from datetime import date, datetime, timedelta, timezone
from pathlib import Path

import httpx
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession

from app.config import settings
from app.exceptions import ProviderError
from app.models.regime_snapshot import RegimeSnapshot
from app.providers.alpaca import AlpacaOHLCVProvider
from app.services import breadth_service, settings_store
from app.services.admin_service import update_setting
from app.services.sentiment_provider_service import _resolve as resolve_llm_config

logger = logging.getLogger(__name__)

_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")

KEY_CONFIG = "regime_monitor_config"
KEY_FUNDAMENTALS = "regime_fundamental_overrides"

# All weights/thresholds are admin-editable via the KEY_CONFIG SystemSetting.
# Default weights sum to 100 (P6 off). SMH is the leading sensor, QQQ confirms.
DEFAULT_CONFIG: dict = {
    "tickers": {
        "leaders": ["SMH"],          # semis — fast early signal
        "confirm": ["QQQ"],          # broad tech — confirmation
        "market": "SPY",
        "breadth": "RSP",            # equal-weight S&P for breadth
        "canary": "NVDA",            # sector lead-stock (optional early warning)
        "hyperscalers": ["GOOGL", "AMZN", "META", "MSFT"],
    },
    "weights": {
        "P1": 12, "P2": 8, "P3": 10, "P4": 8, "P5": 7, "P6": 0,
        "F1": 25, "F2": 15, "F3": 8, "F4": 7,
    },
    "alert_threshold": 65,
    # Observational early-warning blend: a small Combined score = weighted mean of
    # the coincident index and the breadth-divergence early-warning score. Kept
    # separate from the index weights above so the early-warning side stays
    # decoupled until proven. Tunable; need not sum to 1 (normalised).
    "combined_weights": {"coincident": 0.6, "early_warning": 0.4},
    "leader_weight": 2.0,            # SMH counts 2x vs QQQ where both feed a signal
    "rs_lookback": 60,              # trading days for relative-strength / breadth trend
    "fundamental_staleness_days": 80,
}

SIGNAL_LABELS: dict[str, str] = {
    "P1": "Trend break (200-DMA)",
    "P2": "Death cross + slope",
    "P3": "Drawdown from 52w high",
    "P4": "Relative strength SMH/SPY",
    "P5": "Volatility (VIX)",
    "P6": "NVDA canary divergence",
    "F1": "Hyperscaler capex guidance",
    "F2": "Credit spreads (HY OAS)",
    "F3": "Good news, stock down",
    "F4": "Market breadth RSP/SPY",
}

_PRICE_SIGNALS = {"P1", "P2", "P3", "P4", "P6"}


# ---------------------------------------------------------------------------
# Small numeric helpers
# ---------------------------------------------------------------------------

def _clamp(x: float, lo: float = 0.0, hi: float = 100.0) -> float:
    return max(lo, min(hi, x))


def _sma(values: list[float], window: int) -> float | None:
    if len(values) < window:
        return None
    return sum(values[-window:]) / window


def _mean(values: list[float]) -> float | None:
    return sum(values) / len(values) if values else None


def _blend(leader: float | None, confirm: float | None, leader_weight: float) -> float | None:
    """Weighted blend of a leading vs a confirming sub-score (SMH vs QQQ)."""
    parts: list[tuple[float, float]] = []
    if leader is not None:
        parts.append((leader, leader_weight))
    if confirm is not None:
        parts.append((confirm, 1.0))
    if not parts:
        return None
    num = sum(v * w for v, w in parts)
    den = sum(w for _, w in parts)
    return num / den


def band_for(score: float) -> str:
    """Map the 0-100 index onto its label band."""
    if score < 30:
        return "stable"
    if score < 60:
        return "watch"
    if score < 80:
        return "elevated"
    return "breaking"


# ---------------------------------------------------------------------------
# Pure sub-score functions (0 = healthy, 100 = regime breaking). None = no data.
# ---------------------------------------------------------------------------

def _under_200(closes: list[float]) -> float | None:
    sma200 = _sma(closes, 200)
    if sma200 is None:
        return None
    return 100.0 if closes[-1] < sma200 else 0.0


def p1_trend_break(smh: list[float], qqq: list[float], leader_weight: float) -> float | None:
    """Weighted share trading below the 200-DMA (SMH leads, QQQ confirms)."""
    return _blend(_under_200(smh), _under_200(qqq), leader_weight)


def _death_cross(closes: list[float]) -> float | None:
    sma50 = _sma(closes, 50)
    sma200 = _sma(closes, 200)
    if sma50 is None or sma200 is None or len(closes) < 221 or sma200 == 0:
        return None
    gap_pct = (sma50 / sma200 - 1.0) * 100.0
    severity = 0.0 if gap_pct >= 0 else _clamp(-gap_pct * 20.0)  # -5% gap -> 100
    sma200_past = _sma(closes[:-20], 200)
    slope_factor = 1.0
    if sma200_past:
        slope_pct = (sma200 / sma200_past - 1.0) * 100.0
        slope_factor = 1.0 if slope_pct < 0 else 0.5  # damp if 200 still rising
    return severity * slope_factor


def p2_death_cross(smh: list[float], qqq: list[float], leader_weight: float) -> float | None:
    return _blend(_death_cross(smh), _death_cross(qqq), leader_weight)


def _drawdown(closes: list[float]) -> float | None:
    if len(closes) < 30:
        return None
    window = closes[-252:]
    peak = max(window)
    if peak <= 0:
        return None
    dd_pct = (peak - closes[-1]) / peak * 100.0
    return _clamp(dd_pct * 5.0)  # 20% below 52w high -> 100


def p3_drawdown(smh: list[float], qqq: list[float]) -> float | None:
    vals = [v for v in (_drawdown(smh), _drawdown(qqq)) if v is not None]
    return max(vals) if vals else None


def _ratio_trend(a: list[float], b: list[float], lookback: int) -> float | None:
    """Falling a/b (a underperforming b) -> higher score. Flat -> 50."""
    if len(a) < lookback + 1 or len(b) < lookback + 1:
        return None
    if b[-1] == 0 or b[-lookback - 1] == 0:
        return None
    now = a[-1] / b[-1]
    past = a[-lookback - 1] / b[-lookback - 1]
    if past == 0:
        return None
    chg_pct = (now / past - 1.0) * 100.0
    return _clamp(50.0 - chg_pct * 5.0)  # -10% -> 100, +10% -> 0


def p4_relative_strength(smh: list[float], spy: list[float], lookback: int) -> float | None:
    return _ratio_trend(smh, spy, lookback)


def f4_breadth(rsp: list[float], spy: list[float], lookback: int) -> float | None:
    """Narrowing breadth (equal-weight lagging cap-weight) -> RSP/SPY falls -> higher."""
    return _ratio_trend(rsp, spy, lookback)


def p5_volatility(vix: float | None) -> float | None:
    if vix is None:
        return None
    return _clamp((vix - 15.0) / 15.0 * 100.0)  # <=15 -> 0, >=30 -> 100


def f2_credit_spreads(oas_values: list[float]) -> float | None:
    """Percentile rank of the latest HY OAS within the window. Wider = higher."""
    if len(oas_values) < 30:
        return None
    latest = oas_values[-1]
    below = sum(1 for v in oas_values if v <= latest)
    return _clamp(below / len(oas_values) * 100.0)


def p6_canary(nvda: list[float], smh: list[float]) -> float | None:
    """NVDA below its 50-DMA while SMH's trend is still intact = lead divergence."""
    sma50 = _sma(nvda, 50)
    if sma50 is None:
        return None
    nvda_weak = nvda[-1] < sma50
    sma200 = _sma(smh, 200)
    smh_intact = sma200 is not None and smh[-1] > sma200
    if nvda_weak and smh_intact:
        return 100.0
    return 50.0 if nvda_weak else 0.0


# ---------------------------------------------------------------------------
# Aggregation
# ---------------------------------------------------------------------------

def compute_regime_score(sub_scores: dict[str, float | None], weights: dict[str, float]) -> dict:
    """Weighted mean over the *available* signals (weight>0 and score present).

    Missing-data signals drop out of both numerator and denominator and are
    reported with ``available=False``. Contributions sum to the total.
    """
    denom = sum(
        weights.get(sid, 0)
        for sid, score in sub_scores.items()
        if score is not None and weights.get(sid, 0) > 0
    )
    total = 0.0
    breakdown: list[dict] = []
    for sid in SIGNAL_LABELS:
        weight = weights.get(sid, 0)
        if weight <= 0:
            continue
        score = sub_scores.get(sid)
        available = score is not None
        contribution = (score * weight / denom) if (available and denom > 0) else 0.0
        if available:
            total += contribution
        breakdown.append({
            "id": sid,
            "label": SIGNAL_LABELS[sid],
            "sub_score": round(score, 1) if available else None,
            "weight": weight,
            "available": available,
            "contribution": round(contribution, 2),
        })
    return {"total_score": round(total, 1), "band": band_for(total), "breakdown": breakdown}


# ---------------------------------------------------------------------------
# As-of series helpers (for backfill replay)
# ---------------------------------------------------------------------------

Series = list[tuple[date, float]]


def _closes_asof(series: Series, as_of: date) -> list[float]:
    return [v for d, v in series if d <= as_of]


def _value_asof(series: Series | None, as_of: date) -> float | None:
    if not series:
        return None
    vals = [v for d, v in series if d <= as_of]
    return vals[-1] if vals else None


def _window_asof(series: Series | None, as_of: date, years: float) -> list[float]:
    if not series:
        return []
    start = as_of - timedelta(days=int(365 * years))
    return [v for d, v in series if start <= d <= as_of]


def _compute_index(
    prices: dict[str, Series],
    vix_series: Series | None,
    oas_series: Series | None,
    overrides: dict,
    config: dict,
    as_of: date,
) -> dict:
    """Compute the full index result as-of *as_of* from raw series."""
    t = config["tickers"]
    lw = float(config.get("leader_weight", 2.0))
    lb = int(config.get("rs_lookback", 60))

    smh = _closes_asof(prices.get(t["leaders"][0], []), as_of) if t["leaders"] else []
    qqq = _closes_asof(prices.get(t["confirm"][0], []), as_of) if t["confirm"] else []
    spy = _closes_asof(prices.get(t["market"], []), as_of)
    rsp = _closes_asof(prices.get(t["breadth"], []), as_of)
    nvda = _closes_asof(prices.get(t["canary"], []), as_of)
    vix = _value_asof(vix_series, as_of)
    oas = _window_asof(oas_series, as_of, 3)

    sub_scores: dict[str, float | None] = {
        "P1": p1_trend_break(smh, qqq, lw),
        "P2": p2_death_cross(smh, qqq, lw),
        "P3": p3_drawdown(smh, qqq),
        "P4": p4_relative_strength(smh, spy, lb),
        "P5": p5_volatility(vix),
        "P6": p6_canary(nvda, smh),
        "F1": overrides.get("f1_score"),
        "F2": f2_credit_spreads(oas),
        "F3": overrides.get("f3_score"),
        "F4": f4_breadth(rsp, spy, lb),
    }

    result = compute_regime_score(sub_scores, config["weights"])
    result["date"] = as_of.isoformat()
    result["alert_threshold"] = config.get("alert_threshold", 65)
    result["inputs"] = {
        "vix": round(vix, 2) if vix is not None else None,
        "hy_oas": round(oas[-1], 2) if oas else None,
        "fundamentals_fetched_at": overrides.get("fetched_at"),
    }
    return result


# ---------------------------------------------------------------------------
# Config + fundamental-override storage
# ---------------------------------------------------------------------------

async def get_regime_config(db: AsyncSession) -> dict:
    """DEFAULT_CONFIG deep-merged with the stored override (nested for dicts)."""
    cfg = copy.deepcopy(DEFAULT_CONFIG)
    raw = await settings_store.get_value(db, KEY_CONFIG)
    if raw:
        try:
            stored = json.loads(raw)
            for k, v in stored.items():
                if isinstance(v, dict) and isinstance(cfg.get(k), dict):
                    cfg[k].update(v)
                else:
                    cfg[k] = v
        except (TypeError, ValueError):
            logger.warning("Corrupt %s; using defaults", KEY_CONFIG)
    return cfg


async def update_regime_config(db: AsyncSession, updates: dict) -> dict:
    """Merge *updates* into the stored config and persist. Returns the new config."""
    cfg = await get_regime_config(db)
    for k, v in (updates or {}).items():
        if isinstance(v, dict) and isinstance(cfg.get(k), dict):
            cfg[k].update(v)
        else:
            cfg[k] = v
    await update_setting(db, KEY_CONFIG, json.dumps(cfg))
    return cfg


async def get_fundamental_overrides(db: AsyncSession) -> dict:
    """Current F1/F3 override (LLM-proposed or manual). Defaults to neutral 50."""
    raw = await settings_store.get_value(db, KEY_FUNDAMENTALS)
    default = {"f1_score": 50.0, "f3_score": 50.0, "locked": False,
               "reasoning": None, "fetched_at": None, "source": "default"}
    if not raw:
        return default
    try:
        stored = json.loads(raw)
    except (TypeError, ValueError):
        return default
    return {**default, **stored}


async def set_fundamental_overrides(
    db: AsyncSession,
    f1_score: float | None = None,
    f3_score: float | None = None,
    locked: bool | None = None,
) -> dict:
    """Manual override of F1/F3. Setting any value locks out the LLM refresh
    unless ``locked`` is explicitly cleared."""
    current = await get_fundamental_overrides(db)
    if f1_score is not None:
        current["f1_score"] = _clamp(float(f1_score))
    if f3_score is not None:
        current["f3_score"] = _clamp(float(f3_score))
    if locked is not None:
        current["locked"] = bool(locked)
    elif f1_score is not None or f3_score is not None:
        current["locked"] = True
    current["source"] = "manual"
    current["fetched_at"] = datetime.now(timezone.utc).isoformat()
    await update_setting(db, KEY_FUNDAMENTALS, json.dumps(current))
    return current


# ---------------------------------------------------------------------------
# Data fetching: Alpaca prices + FRED macro
# ---------------------------------------------------------------------------

def _price_symbols(config: dict) -> list[str]:
    t = config["tickers"]
    syms = list(t["leaders"]) + list(t["confirm"]) + [t["market"], t["breadth"], t["canary"]]
    seen: list[str] = []
    for s in syms:
        if s and s not in seen:
            seen.append(s)
    return seen


async def _fetch_prices(config: dict, start: date, end: date) -> dict[str, Series]:
    if not settings.alpaca_api_key or not settings.alpaca_api_secret:
        return {}
    provider = AlpacaOHLCVProvider(settings.alpaca_api_key, settings.alpaca_api_secret)
    out: dict[str, Series] = {}
    for sym in _price_symbols(config):
        try:
            bars = await provider.fetch_ohlcv(sym, start, end)
            out[sym] = sorted(((b.date, float(b.close)) for b in bars), key=lambda x: x[0])
        except Exception as exc:
            logger.warning("Regime monitor: price fetch failed for %s: %s", sym, exc)
    return out


async def _fetch_fred_series(series_id: str, start: date, end: date) -> Series | None:
    """Fetch a FRED series as [(date, value)]. None if no API key configured."""
    if not settings.fred_api_key:
        return None
    verify = _CA_BUNDLE if (_CA_BUNDLE and Path(_CA_BUNDLE).exists()) else True
    params = {
        "series_id": series_id,
        "api_key": settings.fred_api_key,
        "file_type": "json",
        "observation_start": start.isoformat(),
        "observation_end": end.isoformat(),
    }
    try:
        async with httpx.AsyncClient(timeout=30, verify=verify) as client:
            resp = await client.get(
                "https://api.stlouisfed.org/fred/series/observations", params=params
            )
            resp.raise_for_status()
            payload = resp.json()
    except Exception as exc:
        logger.warning("Regime monitor: FRED fetch failed for %s: %s", series_id, exc)
        return None

    out: Series = []
    for obs in payload.get("observations", []):
        value = obs.get("value")
        if value in (None, ".", ""):
            continue
        try:
            out.append((date.fromisoformat(obs["date"]), float(value)))
        except (TypeError, ValueError):
            continue
    return sorted(out, key=lambda x: x[0])


# ---------------------------------------------------------------------------
# Snapshot persistence
# ---------------------------------------------------------------------------

async def _upsert_snapshot(db: AsyncSession, result: dict) -> None:
    d = date.fromisoformat(result["date"])
    existing = await db.execute(select(RegimeSnapshot).where(RegimeSnapshot.date == d))
    row = existing.scalar_one_or_none()
    payload = json.dumps(result)
    if row is None:
        db.add(RegimeSnapshot(
            date=d,
            total_score=result["total_score"],
            band=result["band"],
            breakdown_json=payload,
            created_at=datetime.now(timezone.utc),
        ))
    else:
        row.total_score = result["total_score"]
        row.band = result["band"]
        row.breakdown_json = payload


async def _snapshot_count(db: AsyncSession) -> int:
    res = await db.execute(select(func.count()).select_from(RegimeSnapshot))
    return int(res.scalar() or 0)


# ---------------------------------------------------------------------------
# Job entrypoint + reads
# ---------------------------------------------------------------------------

async def update_regime_monitor(db: AsyncSession, backfill_days: int = 90) -> dict:
    """Compute the latest index, persist it, and backfill history on first run.

    Job entrypoint (daily-pipeline step). Best-effort throughout: missing keys or
    a failed source degrade gracefully (signals drop to n/a) rather than abort.
    """
    config = await get_regime_config(db)

    # Refresh the LLM fundamentals if stale (and not manually locked). Best-effort.
    overrides = await get_fundamental_overrides(db)
    if _fundamentals_stale(overrides, config) and not overrides.get("locked"):
        try:
            overrides = await refresh_fundamental_overrides(db, config=config)
        except Exception as exc:
            logger.warning("Regime monitor: fundamentals refresh skipped: %s", exc)

    end = date.today()
    start = end - timedelta(days=400)
    prices = await _fetch_prices(config, start, end)
    vix_series = await _fetch_fred_series("VIXCLS", start, end)
    oas_series = await _fetch_fred_series("BAMLH0A0HYM2", end - timedelta(days=1200), end)

    # Anchor "today" on the latest actual trading day we have prices for.
    leader = config["tickers"]["leaders"][0] if config["tickers"]["leaders"] else None
    leader_series = prices.get(leader or "", [])
    latest_date = leader_series[-1][0] if leader_series else end

    # Early-warning signal: breadth-divergence over the stored universe (leads but
    # noisy). Computed once here so the daily job carries it live, as a SEPARATE
    # score next to the coincident index — not folded into the index weights.
    # Best-effort: a breadth failure must not stop the index update.
    try:
        breadth = await breadth_service.compute_breadth_series(db)
        divergence = breadth_service.compute_divergence_series(breadth, sorted(leader_series))
    except Exception as exc:
        logger.warning("Regime monitor: breadth/divergence skipped: %s", exc)
        divergence = {}
    # As-of lookup: the stored universe (breadth) can lag the live benchmark date
    # by a day or two, so an exact-date match would blank the newest snapshot.
    div_items = sorted(divergence.items())
    cw = config.get("combined_weights") or {"coincident": 0.6, "early_warning": 0.4}

    dates = {latest_date}
    if await _snapshot_count(db) < 5 and leader_series:
        cutoff = end - timedelta(days=backfill_days)
        dates |= {d for d, _ in leader_series if d >= cutoff}

    latest_result: dict | None = None
    for d in sorted(dates):
        result = _compute_index(prices, vix_series, oas_series, overrides, config, d)
        _attach_early_warning(result, _divergence_asof(div_items, d), cw)
        await _upsert_snapshot(db, result)
        latest_result = result

    # Backfill early-warning + combined onto recent existing snapshots (e.g. the
    # index history written before this signal existed) so their 7/30-day trends
    # populate immediately rather than only filling in over the coming weeks.
    if div_items:
        recent = await db.execute(
            select(RegimeSnapshot).where(RegimeSnapshot.date >= end - timedelta(days=120))
        )
        for row in recent.scalars().all():
            try:
                res = json.loads(row.breakdown_json)
            except (TypeError, ValueError):
                continue
            if (res.get("early_warning") or {}).get("score") is not None:
                continue
            _attach_early_warning(res, _divergence_asof(div_items, row.date), cw)
            row.breakdown_json = json.dumps(res)
    await db.commit()

    logger.info(json.dumps({
        "event": "regime_monitor_updated",
        "date": latest_result["date"] if latest_result else None,
        "score": latest_result["total_score"] if latest_result else None,
        "snapshots_written": len(dates),
    }))
    return latest_result or {"available": False, "reason": "no data"}


def _divergence_asof(div_items: list[tuple[date, float]], as_of: date, max_lag_days: int = 7) -> float | None:
    """Latest divergence value on/before ``as_of``, tolerating a small data lag
    between the live benchmark and the stored universe. None if too stale/absent."""
    chosen: tuple[date, float] | None = None
    for d, v in div_items:
        if d <= as_of:
            chosen = (d, v)
        else:
            break
    if chosen is None or (as_of - chosen[0]).days > max_lag_days:
        return None
    return chosen[1]


def _attach_early_warning(result: dict, ew: float | None, weights: dict) -> None:
    """Attach the separate early-warning score and a combined blend to a snapshot.

    ``ew`` is the breadth-divergence value as-of this date (or None). The combined
    score is a normalised weighted mean of the coincident index and the early
    warning — observational, kept apart from the index itself.
    """
    result["early_warning"] = {
        "score": round(ew, 1) if ew is not None else None,
        "band": band_for(ew) if ew is not None else None,
    }
    if ew is None:
        combined = result["total_score"]
    else:
        wc = float(weights.get("coincident", 0.6))
        we = float(weights.get("early_warning", 0.4))
        wsum = (wc + we) or 1.0
        combined = (result["total_score"] * wc + ew * we) / wsum
    result["combined"] = {"score": round(combined, 1), "band": band_for(combined)}


async def _result_at_or_before(db: AsyncSession, target: date) -> dict | None:
    """Parsed snapshot result for the latest date on/before ``target``."""
    res = await db.execute(
        select(RegimeSnapshot.breakdown_json)
        .where(RegimeSnapshot.date <= target)
        .order_by(RegimeSnapshot.date.desc())
        .limit(1)
    )
    raw = res.scalar_one_or_none()
    if raw is None:
        return None
    try:
        return json.loads(raw)
    except (TypeError, ValueError):
        return None


def _delta(curr: float | None, prev: float | None) -> float | None:
    return round(curr - prev, 1) if (curr is not None and prev is not None) else None


async def get_regime_monitor(db: AsyncSession) -> dict:
    """Latest snapshot + 7/30-day trend deltas for the index, early-warning, and
    combined scores. Cheap (a few row reads)."""
    res = await db.execute(
        select(RegimeSnapshot).order_by(RegimeSnapshot.date.desc()).limit(1)
    )
    latest = res.scalar_one_or_none()
    if latest is None:
        return {"available": False, "reason": "not computed yet"}

    try:
        result = json.loads(latest.breakdown_json)
    except (TypeError, ValueError):
        result = {"date": latest.date.isoformat(), "total_score": latest.total_score,
                  "band": latest.band, "breakdown": []}

    r7 = await _result_at_or_before(db, latest.date - timedelta(days=7))
    r30 = await _result_at_or_before(db, latest.date - timedelta(days=30))

    def _nested(r: dict | None, key: str) -> float | None:
        return (r.get(key) or {}).get("score") if r else None

    result["available"] = True
    cur_total = result.get("total_score")
    result["trend"] = {
        "delta_7": _delta(cur_total, (r7 or {}).get("total_score")),
        "delta_30": _delta(cur_total, (r30 or {}).get("total_score")),
    }
    for key in ("early_warning", "combined"):
        block = result.get(key) or {"score": None, "band": None}
        block["delta_7"] = _delta(block.get("score"), _nested(r7, key))
        block["delta_30"] = _delta(block.get("score"), _nested(r30, key))
        result[key] = block
    return result


async def get_regime_history(db: AsyncSession, days: int = 400) -> list[dict]:
    """Daily history of the index, early-warning, and combined scores for the
    score-over-time chart. One point per snapshot date, ascending."""
    cutoff = date.today() - timedelta(days=days)
    res = await db.execute(
        select(RegimeSnapshot)
        .where(RegimeSnapshot.date >= cutoff)
        .order_by(RegimeSnapshot.date.asc())
    )
    out: list[dict] = []
    for row in res.scalars().all():
        try:
            data = json.loads(row.breakdown_json)
        except (TypeError, ValueError):
            data = {}
        out.append({
            "date": row.date.isoformat(),
            "index": row.total_score,
            "early_warning": (data.get("early_warning") or {}).get("score"),
            "combined": (data.get("combined") or {}).get("score"),
        })
    return out


# ---------------------------------------------------------------------------
# F1/F3 via grounded LLM (reuses the configured sentiment provider)
# ---------------------------------------------------------------------------

_CAPEX_PROMPT = """\
You are a markets analyst. Search the web for the MOST RECENT (last reported \
quarter) capital-expenditure (capex) guidance from these hyperscalers: {names}.

For each name, classify the direction of its forward capex/AI-infrastructure \
guidance vs. the prior quarter as exactly one of: "raising", "holding", "cutting".

Also judge the recent "good news, stock down" dynamic: across these names and \
the semiconductor sector, did stocks broadly FALL despite earnings/revenue beats \
in the last reporting season? Answer "yes", "no", or "mixed".

Respond ONLY with a JSON object (no markdown):
{{"capex": {{ {example} }}, "good_news_stock_down": "yes|no|mixed", \
"reasoning": "<2-3 sentences citing the specific guidance you found>"}}
"""


def _fundamentals_stale(overrides: dict, config: dict) -> bool:
    fetched = overrides.get("fetched_at")
    if not fetched:
        return True
    try:
        ts = datetime.fromisoformat(fetched)
    except (TypeError, ValueError):
        return True
    if ts.tzinfo is None:
        ts = ts.replace(tzinfo=timezone.utc)
    max_age = timedelta(days=int(config.get("fundamental_staleness_days", 80)))
    return datetime.now(timezone.utc) - ts > max_age


def _strip_fences(text: str) -> str:
    clean = (text or "").strip()
    if clean.startswith("```"):
        clean = clean.split("\n", 1)[1] if "\n" in clean else clean[3:]
    if clean.endswith("```"):
        clean = clean[:-3]
    return clean.strip()


def _extract_responses_text(response: object) -> str:
    for item in getattr(response, "output", []) or []:
        if getattr(item, "type", None) == "message" and getattr(item, "content", None):
            for block in item.content:
                if getattr(block, "text", None):
                    return block.text
    return ""


async def _call_llm_json(cfg: dict, prompt: str) -> dict:
    """Send one grounded prompt via the configured LLM and parse its JSON reply."""
    provider, model, api_key = cfg["provider"], cfg["model"], cfg["api_key"]
    base_url = cfg.get("base_url")

    if provider == "gemini":
        from google import genai
        from google.genai import types
        client = genai.Client(api_key=api_key)
        resp = await client.aio.models.generate_content(
            model=model,
            contents=prompt,
            config=types.GenerateContentConfig(
                tools=[types.Tool(google_search=types.GoogleSearch())],
                response_mime_type="application/json",
            ),
        )
        return json.loads(_strip_fences(resp.text))

    from openai import AsyncOpenAI
    verify = _CA_BUNDLE if (_CA_BUNDLE and Path(_CA_BUNDLE).exists()) else True
    client = AsyncOpenAI(
        api_key=api_key,
        base_url=base_url or None,
        http_client=httpx.AsyncClient(verify=verify),
    )
    if provider in ("openai", "xai"):
        tool = "web_search_preview" if provider == "openai" else "web_search"
        resp = await client.responses.create(
            model=model,
            tools=[{"type": tool}],
            instructions="Respond with valid JSON only, no markdown fences.",
            input=prompt,
        )
        return json.loads(_strip_fences(_extract_responses_text(resp)))

    # deepseek / generic OpenAI-compatible: no web search, knowledge-based.
    resp = await client.chat.completions.create(
        model=model,
        messages=[{"role": "user", "content": prompt}],
        response_format={"type": "json_object"},
    )
    return json.loads(_strip_fences(resp.choices[0].message.content))


_CAPEX_STATE_SCORES = {"raising": 0.0, "holding": 50.0, "cutting": 100.0}
_GNSD_SCORES = {"yes": 100.0, "mixed": 50.0, "no": 0.0}


async def refresh_fundamental_overrides(
    db: AsyncSession, config: dict | None = None, force: bool = False
) -> dict:
    """Ask the configured LLM to propose F1 (capex) and F3 (earnings reaction).

    Skips (returns current) if a manual override is locked, unless ``force``.
    """
    current = await get_fundamental_overrides(db)
    if current.get("locked") and not force:
        return current

    config = config or await get_regime_config(db)
    cfg = await resolve_llm_config(db)
    if not cfg.get("api_key"):
        raise ProviderError(f"No API key configured for LLM provider '{cfg.get('provider')}'")

    names = config["tickers"]["hyperscalers"]
    example = ", ".join(f'"{n}": "holding"' for n in names)
    prompt = _CAPEX_PROMPT.format(names=", ".join(names), example=example)
    parsed = await _call_llm_json(cfg, prompt)

    capex = parsed.get("capex", {}) if isinstance(parsed, dict) else {}
    scores = [
        _CAPEX_STATE_SCORES[str(capex.get(n, "")).strip().lower()]
        for n in names
        if str(capex.get(n, "")).strip().lower() in _CAPEX_STATE_SCORES
    ]
    f1 = _mean(scores) if scores else 50.0
    gnsd = str(parsed.get("good_news_stock_down", "")).strip().lower()
    f3 = _GNSD_SCORES.get(gnsd, 50.0)

    result = {
        "f1_score": round(f1, 1),
        "f3_score": f3,
        "capex": capex,
        "good_news_stock_down": gnsd or None,
        "reasoning": parsed.get("reasoning") if isinstance(parsed, dict) else None,
        "fetched_at": datetime.now(timezone.utc).isoformat(),
        "locked": False,
        "source": cfg.get("provider"),
    }
    await update_setting(db, KEY_FUNDAMENTALS, json.dumps(result))
    logger.info(json.dumps({"event": "regime_fundamentals_refreshed", "f1": result["f1_score"], "f3": result["f3_score"]}))
    return result