signal-platform/app/services/ticker_universe_service.py

"""Ticker universe discovery and bootstrap service.

Provides a minimal, provider-backed way to populate tracked tickers from
well-known universes (S&P 500, NASDAQ-100, NASDAQ All).
"""

from __future__ import annotations

import asyncio
import json
import logging
import os
import re
from collections.abc import Iterable
from datetime import date, datetime, timezone
from pathlib import Path

import httpx
from sqlalchemy import func, or_, select, update
from sqlalchemy.ext.asyncio import AsyncSession

from app.config import settings
from app.exceptions import ProviderError, ValidationError
from app.models.ticker import Ticker
from app.models.universe_membership import UniverseMembership
from app.services import settings_store, ticker_service

logger = logging.getLogger(__name__)

SUPPORTED_UNIVERSES = {"sp500", "nasdaq100", "nasdaq_all"}
_SYMBOL_PATTERN = re.compile(r"^[A-Z0-9-]{1,10}$")

_SEED_UNIVERSES: dict[str, list[str]] = {
    "sp500": [
        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "GOOG", "BRK-B", "TSLA", "JPM",
        "V", "MA", "UNH", "XOM", "LLY", "AVGO", "COST", "PG", "JNJ", "HD", "MRK", "BAC",
        "ABBV", "PEP", "KO", "ADBE", "NFLX", "CRM", "CSCO", "WMT", "AMD", "TMO", "MCD",
        "ORCL", "ACN", "CVX", "LIN", "DHR", "ABT", "QCOM", "TXN", "PM", "DIS", "INTU",
    ],
    "nasdaq100": [
        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "GOOG", "TSLA", "AVGO", "COST",
        "NFLX", "ADBE", "CSCO", "AMD", "INTU", "QCOM", "AMGN", "TXN", "INTC", "BKNG", "GILD",
        "ISRG", "MDLZ", "ADP", "LRCX", "ADI", "PANW", "SNPS", "CDNS", "KLAC", "MELI", "MU",
        "SBUX", "CSX", "REGN", "VRTX", "MAR", "MNST", "CTAS", "ASML", "PYPL", "AMAT", "NXPI",
    ],
    "nasdaq_all": [
        "AAPL", "MSFT", "NVDA", "AMZN", "META", "GOOGL", "TSLA", "AMD", "INTC", "QCOM", "CSCO",
        "ADBE", "NFLX", "PYPL", "AMAT", "MU", "SBUX", "GILD", "INTU", "BKNG", "ADP", "CTAS",
        "PANW", "SNPS", "CDNS", "LRCX", "KLAC", "MELI", "ASML", "REGN", "VRTX", "MDLZ", "AMGN",
    ],
}

_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
    _CA_BUNDLE_PATH: str | bool = True
else:
    _CA_BUNDLE_PATH = _CA_BUNDLE

# Wikipedia often returns 403 to non-browser UAs; use a normal browser-like
# identity for constituent scrapes (no cookies/login).
_HTTP_HEADERS = {
    "User-Agent": (
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
        "AppleWebKit/537.36 (KHTML, like Gecko) "
        "Chrome/126.0.0.0 Safari/537.36"
    ),
    "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
    "Accept-Language": "en-US,en;q=0.9",
}

# Modern Wikipedia S&P/Nasdaq tables use exchange templates (NyseSymbol /
# NasdaqSymbol) rather than a plain <td><a>SYMBOL</a></td>. Prefer quote URLs
# and template params; keep the legacy cell pattern as a last resort.
_WIKI_SYMBOL_PATTERNS: tuple[re.Pattern[str], ...] = (
    re.compile(r"nyse\.com/quote/XNYS:([A-Za-z0-9.-]{1,10})", re.IGNORECASE),
    re.compile(
        r"nasdaq\.com/market-activity/stocks/([A-Za-z0-9.-]{1,10})",
        re.IGNORECASE,
    ),
    # {{NyseSymbol|BNY}} / {{NasdaqSymbol|AAPL}} rendered data-mw params
    re.compile(
        r'"target":\{"wt":"(?:Nyse|Nasdaq)Symbol"[^}]*\}.*"wt":"([A-Z][A-Z0-9.-]{0,9})"',
        re.IGNORECASE,
    ),
    re.compile(r"<td>\s*<a[^>]*>([A-Z.]{1,10})</a>\s*</td>", re.IGNORECASE),
)


def _extract_wiki_symbols(html: str) -> list[str]:
    """Pull ticker symbols out of a Wikipedia constituents page."""
    found: list[str] = []
    for pattern in _WIKI_SYMBOL_PATTERNS:
        found.extend(pattern.findall(html))
    return found


def _validate_universe(universe: str) -> str:
    normalised = universe.strip().lower()
    if normalised not in SUPPORTED_UNIVERSES:
        supported = ", ".join(sorted(SUPPORTED_UNIVERSES))
        raise ValidationError(f"Unsupported universe '{universe}'. Supported: {supported}")
    return normalised


def _normalise_symbols(symbols: Iterable[str]) -> list[str]:
    deduped: set[str] = set()
    for raw_symbol in symbols:
        symbol = raw_symbol.strip().upper().replace(".", "-")
        if not symbol:
            continue
        if _SYMBOL_PATTERN.fullmatch(symbol) is None:
            continue
        deduped.add(symbol)
    return sorted(deduped)


async def _fetch_wiki_constituent_symbols(
    client: httpx.AsyncClient,
    url: str,
) -> tuple[list[str], str | None]:
    try:
        response = await client.get(url, headers=_HTTP_HEADERS)
    except httpx.HTTPError as exc:
        return [], f"{url}: network error ({type(exc).__name__}: {exc})"

    if response.status_code != 200:
        return [], f"{url}: HTTP {response.status_code}"

    matches = _extract_wiki_symbols(response.text)
    if not matches:
        return [], f"{url}: no symbols parsed"
    return list(matches), None


async def _fetch_nasdaq_trader_symbols(
    client: httpx.AsyncClient,
) -> tuple[list[str], str | None]:
    url = "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt"
    try:
        response = await client.get(url, headers=_HTTP_HEADERS)
    except httpx.HTTPError as exc:
        return [], f"{url}: network error ({type(exc).__name__}: {exc})"

    if response.status_code != 200:
        return [], f"{url}: HTTP {response.status_code}"

    symbols: list[str] = []
    for line in response.text.splitlines():
        if not line or line.startswith("Symbol|") or line.startswith("File Creation Time"):
            continue
        parts = line.split("|")
        if not parts:
            continue
        symbol = parts[0].strip()
        test_issue = parts[6].strip() if len(parts) > 6 else "N"
        if test_issue == "Y":
            continue
        symbols.append(symbol)

    if not symbols:
        return [], f"{url}: no symbols parsed"
    return symbols, None


async def _fetch_universe_symbols_from_public(universe: str) -> tuple[list[str], list[str], str | None]:
    failures: list[str] = []

    sp500_url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
    nasdaq100_url = "https://en.wikipedia.org/wiki/Nasdaq-100"

    async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
        if universe == "sp500":
            symbols, error = await _fetch_wiki_constituent_symbols(client, sp500_url)
            if error:
                failures.append(error)
            else:
                return symbols, failures, "wikipedia_sp500"

        if universe == "nasdaq100":
            symbols, error = await _fetch_wiki_constituent_symbols(client, nasdaq100_url)
            if error:
                failures.append(error)
            else:
                return symbols, failures, "wikipedia_nasdaq100"

        if universe == "nasdaq_all":
            symbols, error = await _fetch_nasdaq_trader_symbols(client)
            if error:
                failures.append(error)
            else:
                return symbols, failures, "nasdaq_trader"

    return [], failures, None


async def _read_cached_symbols(db: AsyncSession, universe: str) -> list[str]:
    key = f"ticker_universe_cache_{universe}"
    setting = await settings_store.get_setting(db, key)
    if setting is None:
        return []

    try:
        payload = json.loads(setting.value)
    except (TypeError, ValueError):
        return []

    if isinstance(payload, dict):
        symbols = payload.get("symbols", [])
    elif isinstance(payload, list):
        symbols = payload
    else:
        symbols = []

    if not isinstance(symbols, list):
        return []

    return _normalise_symbols([str(symbol) for symbol in symbols])


async def _write_cached_symbols(
    db: AsyncSession,
    universe: str,
    symbols: list[str],
    source: str,
) -> None:
    key = f"ticker_universe_cache_{universe}"
    payload = {
        "symbols": symbols,
        "source": source,
        "updated_at": datetime.now(timezone.utc).isoformat(),
    }

    await settings_store.upsert_setting(db, key, json.dumps(payload))
    await db.commit()


async def fetch_universe_symbols(
    db: AsyncSession,
    universe: str,
) -> tuple[list[str], str]:
    """Fetch and normalise symbols for a supported universe with fallbacks.

    Fallback order:
    1) Free public sources (Wikipedia/NASDAQ trader)
    2) Cached snapshot in SystemSetting
    3) Built-in seed symbols

    Returns ``(symbols, source_label)`` so bootstrap UI can show where the
    list came from (important when the public source fails and a stale cache
    still lists BK instead of BNY).

    The seeds are representative, not complete, so a *fresh* install whose
    public source is down bootstraps a partial universe. A warm instance is
    unaffected — it falls through to its cached snapshot.
    """
    normalised_universe = _validate_universe(universe)
    failures: list[str] = []

    public_symbols, public_failures, public_source = await _fetch_universe_symbols_from_public(normalised_universe)
    failures.extend(public_failures)
    cleaned_public = _normalise_symbols(public_symbols)
    if cleaned_public:
        await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
        return cleaned_public, public_source or "public"

    cached_symbols = await _read_cached_symbols(db, normalised_universe)
    if cached_symbols:
        logger.warning(
            "Using cached universe symbols for %s because live fetch failed: %s",
            normalised_universe,
            "; ".join(failures[:3]),
        )
        return cached_symbols, "cache"

    seed_symbols = _normalise_symbols(_SEED_UNIVERSES.get(normalised_universe, []))
    if seed_symbols:
        logger.warning(
            "Using built-in seed symbols for %s because live/cache fetch failed: %s",
            normalised_universe,
            "; ".join(failures[:3]),
        )
        return seed_symbols, "seed"

    reason = "; ".join(failures[:6]) if failures else "no provider returned symbols"
    raise ProviderError(f"Universe '{normalised_universe}' returned no valid symbols. Attempts: {reason}")


async def _fetch_alpaca_asset_names() -> dict[str, str]:
    """One Alpaca Trading-API call → {internal_symbol: company_name} for all US
    equities. Tries paper and live endpoints so it works with either key type."""
    if not settings.alpaca_api_key or not settings.alpaca_api_secret:
        raise ValidationError("Alpaca API credentials are required to backfill names")

    from alpaca.trading.client import TradingClient
    from alpaca.trading.enums import AssetClass, AssetStatus
    from alpaca.trading.requests import GetAssetsRequest

    req = GetAssetsRequest(status=AssetStatus.ACTIVE, asset_class=AssetClass.US_EQUITY)
    last_err: Exception | None = None
    for paper in (True, False):
        try:
            client = TradingClient(settings.alpaca_api_key, settings.alpaca_api_secret, paper=paper)
            assets = await asyncio.to_thread(client.get_all_assets, req)
            names: dict[str, str] = {}
            for asset in assets:
                sym = getattr(asset, "symbol", None)
                nm = getattr(asset, "name", None)
                if sym and nm:
                    names[sym.replace(".", "-").upper()] = nm  # BRK.B → BRK-B
            if names:
                return names
        except Exception as exc:  # noqa: BLE001 — try the other endpoint
            last_err = exc

    raise ProviderError(f"Failed to fetch asset names from Alpaca: {last_err}")


async def backfill_ticker_names(db: AsyncSession, *, only_missing: bool = True) -> dict[str, int]:
    """Fill Ticker.name from Alpaca in a single request for the whole universe."""
    result = await db.execute(select(Ticker))
    tickers = list(result.scalars().all())
    targets = [t for t in tickers if not t.name] if only_missing else tickers
    if not targets:
        return {"updated": 0, "checked": 0, "unmatched": 0}

    names = await _fetch_alpaca_asset_names()
    updated = 0
    for ticker in targets:
        nm = names.get(ticker.symbol.upper())
        if nm and nm != ticker.name:
            ticker.name = nm[:120]
            updated += 1
    await db.commit()
    return {"updated": updated, "checked": len(targets), "unmatched": len(targets) - updated}


# Sources that represent an actual look at the index today. A ``cache`` or
# ``seed`` result means the live fetch FAILED — the seed lists are ~44 symbols,
# so treating one as an observation would close ~460 memberships on a network
# blip. Membership is only ever recorded from a live read.
LIVE_UNIVERSE_SOURCES = frozenset({
    "wikipedia_sp500", "wikipedia_nasdaq100", "nasdaq_trader", "public",
})


async def members_on(db: AsyncSession, universe: str, as_of: date) -> list[str]:
    """Which symbols were observed in *universe* on *as_of*, alphabetically.

    The point-in-time question item 3 exists to answer, with the interval
    boundaries written down once as code rather than re-derived at each call
    site:

        member on D  <=>  first_seen_on <= D < absent_since (or absent_since IS NULL)

    Lower bound inclusive, upper bound exclusive. ``absent_since`` is the first
    observation that found the symbol *gone*, so on that date it was already
    absent.

    **This is observation history, not index history.** A date before the first
    successful live sync returns an empty list — that is "we were not looking",
    not "the index was empty", and the caller has to know the difference. The
    reconstructed removals in ``scripts/data/sp500_removals.json`` are research
    data and deliberately do not seed this table.
    """
    rows = await db.execute(
        select(Ticker.symbol)
        .join(UniverseMembership, UniverseMembership.ticker_id == Ticker.id)
        .where(
            UniverseMembership.universe == universe,
            UniverseMembership.first_seen_on <= as_of,
            or_(
                UniverseMembership.absent_since.is_(None),
                UniverseMembership.absent_since > as_of,
            ),
        )
        .order_by(Ticker.symbol)
    )
    return list(rows.scalars().all())


async def _observe_membership(
    db: AsyncSession, universe: str, present: set[str]
) -> dict[str, int]:
    """Record what a live universe fetch just saw. Returns opened/closed counts.

    Runs on every sync, not only when pruning: the scheduled job uses
    ``prune_missing=False``, so membership recorded only on manual prunes would
    be full of holes. Recording membership and deciding what to scan are
    separate concerns, and this is the recording one — it never touches
    ``untracked_on``, ``scan_disabled_on`` or ``tracking_source``, and it applies
    to manually tracked symbols exactly as it does to index-owned ones.

    Intervals are append-only. A symbol that leaves and returns gets a second
    row, so a point-in-time query for the gap answers "not a member"; reopening
    the first row would erase that.

    Dates come from the database (``current_date``), matching every other date
    this module writes. ``first_seen_on`` is the first *observation*, not an
    index effective date — anything earlier is unknown and is not guessed.
    """
    rows = (await db.execute(
        select(Ticker.id, Ticker.symbol).where(Ticker.symbol.in_(present))
    )).all()
    ticker_ids = {symbol: tid for tid, symbol in rows}

    open_rows = (await db.execute(
        select(UniverseMembership.ticker_id, Ticker.symbol)
        .join(Ticker, Ticker.id == UniverseMembership.ticker_id)
        .where(
            UniverseMembership.universe == universe,
            UniverseMembership.absent_since.is_(None),
        )
    )).all()
    open_symbols = {symbol for _, symbol in open_rows}

    opened = 0
    for symbol in sorted(present - open_symbols):
        ticker_id = ticker_ids.get(symbol)
        if ticker_id is None:
            continue
        db.add(UniverseMembership(
            ticker_id=ticker_id,
            universe=universe,
            first_seen_on=func.current_date(),
        ))
        opened += 1

    departed_ids = [tid for tid, symbol in open_rows if symbol not in present]
    closed = 0
    if departed_ids:
        result = await db.execute(
            update(UniverseMembership)
            .where(
                UniverseMembership.universe == universe,
                UniverseMembership.absent_since.is_(None),
                UniverseMembership.ticker_id.in_(departed_ids),
            )
            .values(absent_since=func.current_date())
        )
        closed = int(result.rowcount or 0)

    return {"opened": opened, "closed": closed}


async def bootstrap_universe(
    db: AsyncSession,
    universe: str,
    *,
    prune_missing: bool = False,
) -> dict[str, int | str]:
    """Upsert ticker universe into tracked tickers.

    ``prune_missing`` marks symbols no longer in the universe as untracked; it
    never deletes them. "Not in today's scanner" and "not stored for historical
    analysis" are different states, and conflating them is what makes a universe
    quietly become the survivors. Symbols that have rejoined the universe have
    the mark cleared and return to the live path with their history intact.

    Returns summary counts for added / already-tracked / untracked / retracked.
    """
    normalised_universe = _validate_universe(universe)
    symbols, source = await fetch_universe_symbols(db, normalised_universe)

    # A cache or seed result means the live fetch FAILED. The seed lists are ~44
    # symbols against a ~500-symbol universe, so treating one as the truth would
    # untrack the entire index on a single network blip. Nothing that acts on
    # "this symbol is/isn't in the index" may run on fallback data: not pruning,
    # not clearing a prune flag, not recording membership. Adding is the one
    # exception — it destroys nothing and is what lets a cold start bootstrap
    # from seeds.
    observed = source in LIVE_UNIVERSE_SOURCES

    existing_rows = await db.execute(select(Ticker.symbol))
    existing_symbols = set(existing_rows.scalars().all())
    target_symbols = set(symbols)

    symbols_to_add = sorted(target_symbols - existing_symbols)

    # Only index-owned rows are prunable. A manual addition is by definition not
    # in the constituent list, so without this filter every hand-added symbol
    # would be swept by any prune.
    prunable = set() if not (prune_missing and observed) else set((
        await db.execute(
            select(Ticker.symbol).where(
                Ticker.symbol.in_(existing_symbols - target_symbols),
                Ticker.tracking_source == ticker_service.SOURCE_INDEX,
            )
        )
    ).scalars().all())
    symbols_to_untrack = sorted(prunable)
    kept_manual = (
        sorted((existing_symbols - target_symbols) - prunable)
        if (prune_missing and observed) else []
    )

    for symbol in symbols_to_add:
        db.add(Ticker(symbol=symbol, tracking_source=ticker_service.SOURCE_INDEX))

    # A symbol that has returned to the index must rejoin the live path. It is
    # already in ``existing_symbols``, so it is neither added nor untracked here
    # — without this it would keep the flag from its earlier removal and stay
    # invisible to the scanner forever, with nothing to show why.
    #
    # Clears the prune flag ONLY. ``scan_disabled_on`` is the operator's own
    # decision and no automated run may undo it: a symbol someone switched off
    # by hand must stay off across a prune -> re-entry cycle. Nor does this touch
    # ``tracking_source`` — a manual row that joins the index stays manual, or it
    # would lose its protection the next time it left.
    retracked_count = 0
    if observed:
        retracked = await db.execute(
            update(Ticker)
            .where(Ticker.symbol.in_(target_symbols), Ticker.untracked_on.is_not(None))
            .values(untracked_on=None)
        )
        retracked_count = int(retracked.rowcount or 0)

    untracked_count = 0
    if symbols_to_untrack:
        # Marked, never deleted. The delete this replaced cascaded through
        # OHLCV, setups and scores, which is precisely the history a
        # survivorship-honest backtest needs — see migration 035. Rows already
        # carrying a mark keep their original date rather than being re-stamped
        # on every bootstrap.
        #
        # Stamped by the database, not date.today(): active_only compares this
        # against func.current_date(), and a process in a timezone ahead of the
        # database would otherwise write a date that reads as "effective
        # tomorrow" and leave the symbol in the scanner for a day. Same clock on
        # both sides by construction.
        result = await db.execute(
            update(Ticker)
            .where(Ticker.symbol.in_(symbols_to_untrack), Ticker.untracked_on.is_(None))
            .values(untracked_on=func.current_date())
        )
        untracked_count = int(result.rowcount or 0)

    # Membership is observed on every live sync, independent of pruning and of
    # who owns the row. A failed fetch must never close a membership.
    membership = (
        await _observe_membership(db, normalised_universe, target_symbols)
        if observed
        else {"opened": 0, "closed": 0}
    )

    await db.commit()

    # Best-effort: fill company names for any tickers still missing one. Never let
    # a name-fetch hiccup fail the bootstrap itself.
    try:
        await backfill_ticker_names(db, only_missing=True)
    except Exception:  # noqa: BLE001
        logger.warning("Ticker name backfill failed during bootstrap", exc_info=True)

    return {
        "universe": normalised_universe,
        "source": source,
        "total_universe_symbols": len(symbols),
        "added": len(symbols_to_add),
        "already_tracked": len(target_symbols & existing_symbols),
        # Always 0. Kept so a caller reading this key is told plainly that
        # pruning no longer destroys rows, rather than seeing the old key vanish
        # and assuming nothing happened — or worse, seeing a count under
        # "deleted" that now means something else entirely.
        "deleted": 0,
        "untracked": untracked_count,
        "untracked_symbols": symbols_to_untrack[:50],
        # Symbols that rejoined the index and were returned to the live path.
        "retracked": retracked_count,
        # Absent from the universe but manually owned, so a prune left them
        # alone. The operator asked for these by hand.
        "kept_manual": kept_manual[:50],
        "kept_manual_count": len(kept_manual),
        "added_symbols": symbols_to_add[:50],
        # Membership observation is skipped entirely on a cache/seed fallback,
        # so the caller can tell "nothing changed" from "we never looked".
        "membership_observed": observed,
        "membership_opened": membership["opened"],
        "membership_closed": membership["closed"],
        "prune_note": (
            "Pruning marks untracked_on and never deletes: the rows keep their "
            "bars, setups and scores so historical backtests stay honest. "
            "Untracked symbols leave the scanner, rankings and ingestion. "
            "Manually added symbols are never pruned."
        ),
    }