signal-platform/app/services/ticker_universe_service.py

"""Ticker universe discovery and bootstrap service.

Provides a minimal, provider-backed way to populate tracked tickers from
well-known universes (S&P 500, NASDAQ-100, NASDAQ All).
"""

from __future__ import annotations

import asyncio
import json
import logging
import os
import re
from collections.abc import Iterable
from datetime import datetime, timezone
from pathlib import Path

import httpx
from sqlalchemy import delete, select
from sqlalchemy.ext.asyncio import AsyncSession

from app.config import settings
from app.exceptions import ProviderError, ValidationError
from app.models.ticker import Ticker
from app.services import settings_store

logger = logging.getLogger(__name__)

SUPPORTED_UNIVERSES = {"sp500", "nasdaq100", "nasdaq_all"}
_SYMBOL_PATTERN = re.compile(r"^[A-Z0-9-]{1,10}$")

_SEED_UNIVERSES: dict[str, list[str]] = {
    "sp500": [
        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "GOOG", "BRK-B", "TSLA", "JPM",
        "V", "MA", "UNH", "XOM", "LLY", "AVGO", "COST", "PG", "JNJ", "HD", "MRK", "BAC",
        "ABBV", "PEP", "KO", "ADBE", "NFLX", "CRM", "CSCO", "WMT", "AMD", "TMO", "MCD",
        "ORCL", "ACN", "CVX", "LIN", "DHR", "ABT", "QCOM", "TXN", "PM", "DIS", "INTU",
    ],
    "nasdaq100": [
        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "GOOG", "TSLA", "AVGO", "COST",
        "NFLX", "ADBE", "CSCO", "AMD", "INTU", "QCOM", "AMGN", "TXN", "INTC", "BKNG", "GILD",
        "ISRG", "MDLZ", "ADP", "LRCX", "ADI", "PANW", "SNPS", "CDNS", "KLAC", "MELI", "MU",
        "SBUX", "CSX", "REGN", "VRTX", "MAR", "MNST", "CTAS", "ASML", "PYPL", "AMAT", "NXPI",
    ],
    "nasdaq_all": [
        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "TSLA", "AMD", "INTC", "QCOM", "CSCO",
        "ADBE", "NFLX", "PYPL", "AMAT", "MU", "SBUX", "GILD", "INTU", "BKNG", "ADP", "CTAS",
        "PANW", "SNPS", "CDNS", "LRCX", "KLAC", "MELI", "ASML", "REGN", "VRTX", "MDLZ", "AMGN",
    ],
}

_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
    _CA_BUNDLE_PATH: str | bool = True
else:
    _CA_BUNDLE_PATH = _CA_BUNDLE

# Wikipedia often returns 403 to non-browser UAs; use a normal browser-like
# identity for constituent scrapes (no cookies/login).
_HTTP_HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/126.0.0.0 Safari/537.36"
    ),
    "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}

# Modern Wikipedia S&P/Nasdaq tables use exchange templates (NyseSymbol /
# NasdaqSymbol) rather than a plain <td><a>SYMBOL</a></td>. Prefer quote URLs
# and template params; keep the legacy cell pattern as a last resort.
_WIKI_SYMBOL_PATTERNS: tuple[re.Pattern[str], ...] = (
    re.compile(r"nyse\.com/quote/XNYS:([A-Za-z0-9.-]{1,10})", re.IGNORECASE),
    re.compile(
        r"nasdaq\.com/market-activity/stocks/([A-Za-z0-9.-]{1,10})",
        re.IGNORECASE,
    ),
    # {{NyseSymbol|BNY}} / {{NasdaqSymbol|AAPL}} rendered data-mw params
    re.compile(
        r'"target":\{"wt":"(?:Nyse|Nasdaq)Symbol"[^}]*\}.*"wt":"([A-Z][A-Z0-9.-]{0,9})"',
        re.IGNORECASE,
    ),
    re.compile(r"<td>\s*<a[^>]*>([A-Z.]{1,10})</a>\s*</td>", re.IGNORECASE),
)


def _extract_wiki_symbols(html: str) -> list[str]:
    """Pull ticker symbols out of a Wikipedia constituents page."""
    found: list[str] = []
    for pattern in _WIKI_SYMBOL_PATTERNS:
        found.extend(pattern.findall(html))
    return found


def _validate_universe(universe: str) -> str:
    normalised = universe.strip().lower()
    if normalised not in SUPPORTED_UNIVERSES:
        supported = ", ".join(sorted(SUPPORTED_UNIVERSES))
        raise ValidationError(f"Unsupported universe '{universe}'. Supported: {supported}")
    return normalised


def _normalise_symbols(symbols: Iterable[str]) -> list[str]:
    deduped: set[str] = set()
    for raw_symbol in symbols:
        symbol = raw_symbol.strip().upper().replace(".", "-")
        if not symbol:
            continue
        if _SYMBOL_PATTERN.fullmatch(symbol) is None:
            continue
        deduped.add(symbol)
    return sorted(deduped)


async def _fetch_wiki_constituent_symbols(
    client: httpx.AsyncClient,
    url: str,
) -> tuple[list[str], str | None]:
    try:
        response = await client.get(url, headers=_HTTP_HEADERS)
    except httpx.HTTPError as exc:
        return [], f"{url}: network error ({type(exc).__name__}: {exc})"

    if response.status_code != 200:
        return [], f"{url}: HTTP {response.status_code}"

    matches = _extract_wiki_symbols(response.text)
    if not matches:
        return [], f"{url}: no symbols parsed"
    return list(matches), None


async def _fetch_nasdaq_trader_symbols(
    client: httpx.AsyncClient,
) -> tuple[list[str], str | None]:
    url = "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt"
    try:
        response = await client.get(url, headers=_HTTP_HEADERS)
    except httpx.HTTPError as exc:
        return [], f"{url}: network error ({type(exc).__name__}: {exc})"

    if response.status_code != 200:
        return [], f"{url}: HTTP {response.status_code}"

    symbols: list[str] = []
    for line in response.text.splitlines():
        if not line or line.startswith("Symbol|") or line.startswith("File Creation Time"):
            continue
        parts = line.split("|")
        if not parts:
            continue
        symbol = parts[0].strip()
        test_issue = parts[6].strip() if len(parts) > 6 else "N"
        if test_issue == "Y":
            continue
        symbols.append(symbol)

    if not symbols:
        return [], f"{url}: no symbols parsed"
    return symbols, None


async def _fetch_universe_symbols_from_public(universe: str) -> tuple[list[str], list[str], str | None]:
    failures: list[str] = []

    sp500_url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
    nasdaq100_url = "https://en.wikipedia.org/wiki/Nasdaq-100"

    async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
        if universe == "sp500":
            symbols, error = await _fetch_wiki_constituent_symbols(client, sp500_url)
            if error:
                failures.append(error)
            else:
                return symbols, failures, "wikipedia_sp500"

        if universe == "nasdaq100":
            symbols, error = await _fetch_wiki_constituent_symbols(client, nasdaq100_url)
            if error:
                failures.append(error)
            else:
                return symbols, failures, "wikipedia_nasdaq100"

        if universe == "nasdaq_all":
            symbols, error = await _fetch_nasdaq_trader_symbols(client)
            if error:
                failures.append(error)
            else:
                return symbols, failures, "nasdaq_trader"

    return [], failures, None


async def _read_cached_symbols(db: AsyncSession, universe: str) -> list[str]:
    key = f"ticker_universe_cache_{universe}"
    setting = await settings_store.get_setting(db, key)
    if setting is None:
        return []

    try:
        payload = json.loads(setting.value)
    except (TypeError, ValueError):
        return []

    if isinstance(payload, dict):
        symbols = payload.get("symbols", [])
    elif isinstance(payload, list):
        symbols = payload
    else:
        symbols = []

    if not isinstance(symbols, list):
        return []

    return _normalise_symbols([str(symbol) for symbol in symbols])


async def _write_cached_symbols(
    db: AsyncSession,
    universe: str,
    symbols: list[str],
    source: str,
) -> None:
    key = f"ticker_universe_cache_{universe}"
    payload = {
        "symbols": symbols,
        "source": source,
        "updated_at": datetime.now(timezone.utc).isoformat(),
    }

    await settings_store.upsert_setting(db, key, json.dumps(payload))
    await db.commit()


async def fetch_universe_symbols(
    db: AsyncSession,
    universe: str,
) -> tuple[list[str], str]:
    """Fetch and normalise symbols for a supported universe with fallbacks.

    Fallback order:
    1) Free public sources (Wikipedia/NASDAQ trader)
    2) Cached snapshot in SystemSetting
    3) Built-in seed symbols

    Returns ``(symbols, source_label)`` so bootstrap UI can show where the
    list came from (important when the public source fails and a stale cache
    still lists BK instead of BNY).

    The seeds are representative, not complete, so a *fresh* install whose
    public source is down bootstraps a partial universe. A warm instance is
    unaffected — it falls through to its cached snapshot.
    """
    normalised_universe = _validate_universe(universe)
    failures: list[str] = []

    public_symbols, public_failures, public_source = await _fetch_universe_symbols_from_public(normalised_universe)
    failures.extend(public_failures)
    cleaned_public = _normalise_symbols(public_symbols)
    if cleaned_public:
        await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
        return cleaned_public, public_source or "public"

    cached_symbols = await _read_cached_symbols(db, normalised_universe)
    if cached_symbols:
        logger.warning(
            "Using cached universe symbols for %s because live fetch failed: %s",
            normalised_universe,
            "; ".join(failures[:3]),
        )
        return cached_symbols, "cache"

    seed_symbols = _normalise_symbols(_SEED_UNIVERSES.get(normalised_universe, []))
    if seed_symbols:
        logger.warning(
            "Using built-in seed symbols for %s because live/cache fetch failed: %s",
            normalised_universe,
            "; ".join(failures[:3]),
        )
        return seed_symbols, "seed"

    reason = "; ".join(failures[:6]) if failures else "no provider returned symbols"
    raise ProviderError(f"Universe '{normalised_universe}' returned no valid symbols. Attempts: {reason}")


async def _fetch_alpaca_asset_names() -> dict[str, str]:
    """One Alpaca Trading-API call → {internal_symbol: company_name} for all US
    equities. Tries paper and live endpoints so it works with either key type."""
    if not settings.alpaca_api_key or not settings.alpaca_api_secret:
        raise ValidationError("Alpaca API credentials are required to backfill names")

    from alpaca.trading.client import TradingClient
    from alpaca.trading.enums import AssetClass, AssetStatus
    from alpaca.trading.requests import GetAssetsRequest

    req = GetAssetsRequest(status=AssetStatus.ACTIVE, asset_class=AssetClass.US_EQUITY)
    last_err: Exception | None = None
    for paper in (True, False):
        try:
            client = TradingClient(settings.alpaca_api_key, settings.alpaca_api_secret, paper=paper)
            assets = await asyncio.to_thread(client.get_all_assets, req)
            names: dict[str, str] = {}
            for asset in assets:
                sym = getattr(asset, "symbol", None)
                nm = getattr(asset, "name", None)
                if sym and nm:
                    names[sym.replace(".", "-").upper()] = nm  # BRK.B → BRK-B
            if names:
                return names
        except Exception as exc:  # noqa: BLE001 — try the other endpoint
            last_err = exc

    raise ProviderError(f"Failed to fetch asset names from Alpaca: {last_err}")


async def backfill_ticker_names(db: AsyncSession, *, only_missing: bool = True) -> dict[str, int]:
    """Fill Ticker.name from Alpaca in a single request for the whole universe."""
    result = await db.execute(select(Ticker))
    tickers = list(result.scalars().all())
    targets = [t for t in tickers if not t.name] if only_missing else tickers
    if not targets:
        return {"updated": 0, "checked": 0, "unmatched": 0}

    names = await _fetch_alpaca_asset_names()
    updated = 0
    for ticker in targets:
        nm = names.get(ticker.symbol.upper())
        if nm and nm != ticker.name:
            ticker.name = nm[:120]
            updated += 1
    await db.commit()
    return {"updated": updated, "checked": len(targets), "unmatched": len(targets) - updated}


async def bootstrap_universe(
    db: AsyncSession,
    universe: str,
    *,
    prune_missing: bool = False,
) -> dict[str, int | str]:
    """Upsert ticker universe into tracked tickers.

    Returns summary counts for added/existing/deleted symbols.
    """
    normalised_universe = _validate_universe(universe)
    symbols, source = await fetch_universe_symbols(db, normalised_universe)

    existing_rows = await db.execute(select(Ticker.symbol))
    existing_symbols = set(existing_rows.scalars().all())
    target_symbols = set(symbols)

    symbols_to_add = sorted(target_symbols - existing_symbols)
    symbols_to_delete = sorted(existing_symbols - target_symbols) if prune_missing else []

    for symbol in symbols_to_add:
        db.add(Ticker(symbol=symbol))

    deleted_count = 0
    skipped_delisted: list[str] = []
    if symbols_to_delete:
        # A delisted row was retained on purpose — its price history is exactly
        # what a survivorship-honest backtest needs, and the delete cascades it
        # away. Pruning must not undo that. (Pruning a symbol that is merely no
        # longer an index constituent still destroys history; that needs a
        # tracked/membership state separate from delisting.)
        protected = (
            await db.execute(
                select(Ticker.symbol).where(
                    Ticker.symbol.in_(symbols_to_delete),
                    Ticker.delisted_on.is_not(None),
                )
            )
        ).scalars().all()
        skipped_delisted = sorted(protected)
        deletable = [s for s in symbols_to_delete if s not in set(protected)]
        if deletable:
            result = await db.execute(delete(Ticker).where(Ticker.symbol.in_(deletable)))
            deleted_count = int(result.rowcount or 0)

    await db.commit()

    # Best-effort: fill company names for any tickers still missing one. Never let
    # a name-fetch hiccup fail the bootstrap itself.
    try:
        await backfill_ticker_names(db, only_missing=True)
    except Exception:  # noqa: BLE001
        logger.warning("Ticker name backfill failed during bootstrap", exc_info=True)

    return {
        "universe": normalised_universe,
        "source": source,
        "total_universe_symbols": len(symbols),
        "added": len(symbols_to_add),
        "already_tracked": len(target_symbols & existing_symbols),
        "deleted": deleted_count,
        "added_symbols": symbols_to_add[:50],
        # Delisted rows a prune declined to destroy, so the caller can see the
        # count did not match what they asked to remove.
        "kept_delisted": skipped_delisted[:50],
        "kept_delisted_count": len(skipped_delisted),
    }