Fix S&P 500 universe parse so renames like BNY are discovered.

Wikipedia no longer uses plain symbol table cells; parse exchange links and NyseSymbol templates, surface the list source in bootstrap results, and keep legacy cell parsing as a fallback.

app/services/ticker_universe_service.py

@@ -55,6 +55,43 @@ if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
 else:
     _CA_BUNDLE_PATH = _CA_BUNDLE
 
+# Wikipedia often returns 403 to non-browser UAs; use a normal browser-like
+# identity for constituent scrapes (no cookies/login).
+_HTTP_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/126.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+# Modern Wikipedia S&P/Nasdaq tables use exchange templates (NyseSymbol /
+# NasdaqSymbol) rather than a plain <td><a>SYMBOL</a></td>. Prefer quote URLs
+# and template params; keep the legacy cell pattern as a last resort.
+_WIKI_SYMBOL_PATTERNS: tuple[re.Pattern[str], ...] = (
+    re.compile(r"nyse\.com/quote/XNYS:([A-Za-z0-9.-]{1,10})", re.IGNORECASE),
+    re.compile(
+        r"nasdaq\.com/market-activity/stocks/([A-Za-z0-9.-]{1,10})",
+        re.IGNORECASE,
+    ),
+    # {{NyseSymbol|BNY}} / {{NasdaqSymbol|AAPL}} rendered data-mw params
+    re.compile(
+        r'"target":\{"wt":"(?:Nyse|Nasdaq)Symbol"[^}]*\}.*"wt":"([A-Z][A-Z0-9.-]{0,9})"',
+        re.IGNORECASE,
+    ),
+    re.compile(r"<td>\s*<a[^>]*>([A-Z.]{1,10})</a>\s*</td>", re.IGNORECASE),
+)
+
+
+def _extract_wiki_symbols(html: str) -> list[str]:
+    """Pull ticker symbols out of a Wikipedia constituents page."""
+    found: list[str] = []
+    for pattern in _WIKI_SYMBOL_PATTERNS:
+        found.extend(pattern.findall(html))
+    return found
+
 
 def _validate_universe(universe: str) -> str:
     normalised = universe.strip().lower()
@@ -186,20 +223,19 @@ async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]:
     raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'")
 
 
-async def _fetch_html_symbols(
+async def _fetch_wiki_constituent_symbols(
     client: httpx.AsyncClient,
     url: str,
-    pattern: str,
 ) -> tuple[list[str], str | None]:
     try:
-        response = await client.get(url)
+        response = await client.get(url, headers=_HTTP_HEADERS)
     except httpx.HTTPError as exc:
         return [], f"{url}: network error ({type(exc).__name__}: {exc})"
 
     if response.status_code != 200:
         return [], f"{url}: HTTP {response.status_code}"
 
-    matches = re.findall(pattern, response.text, flags=re.IGNORECASE)
+    matches = _extract_wiki_symbols(response.text)
     if not matches:
         return [], f"{url}: no symbols parsed"
     return list(matches), None
@@ -210,7 +246,7 @@ async def _fetch_nasdaq_trader_symbols(
 ) -> tuple[list[str], str | None]:
     url = "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt"
     try:
-        response = await client.get(url)
+        response = await client.get(url, headers=_HTTP_HEADERS)
     except httpx.HTTPError as exc:
         return [], f"{url}: network error ({type(exc).__name__}: {exc})"
 
@@ -240,18 +276,17 @@ async def _fetch_universe_symbols_from_public(universe: str) -> tuple[list[str],
 
     sp500_url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
     nasdaq100_url = "https://en.wikipedia.org/wiki/Nasdaq-100"
-    wiki_symbol_pattern = r"<td>\s*<a[^>]*>([A-Z.]{1,10})</a>\s*</td>"
 
     async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
         if universe == "sp500":
-            symbols, error = await _fetch_html_symbols(client, sp500_url, wiki_symbol_pattern)
+            symbols, error = await _fetch_wiki_constituent_symbols(client, sp500_url)
             if error:
                 failures.append(error)
             else:
                 return symbols, failures, "wikipedia_sp500"
 
         if universe == "nasdaq100":
-            symbols, error = await _fetch_html_symbols(client, nasdaq100_url, wiki_symbol_pattern)
+            symbols, error = await _fetch_wiki_constituent_symbols(client, nasdaq100_url)
             if error:
                 failures.append(error)
             else:
@@ -308,7 +343,10 @@ async def _write_cached_symbols(
     await db.commit()
 
 
-async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
+async def fetch_universe_symbols(
+    db: AsyncSession,
+    universe: str,
+) -> tuple[list[str], str]:
     """Fetch and normalise symbols for a supported universe with fallbacks.
 
     Fallback order:
@@ -316,6 +354,10 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
     2) FMP endpoints (if available)
     3) Cached snapshot in SystemSetting
     4) Built-in seed symbols
+
+    Returns ``(symbols, source_label)`` so bootstrap UI can show where the
+    list came from (important when Wikipedia/FMP fail and a stale cache still
+    lists BK instead of BNY).
     """
     normalised_universe = _validate_universe(universe)
     failures: list[str] = []
@@ -325,14 +367,14 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
     cleaned_public = _normalise_symbols(public_symbols)
     if cleaned_public:
         await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
-        return cleaned_public
+        return cleaned_public, public_source or "public"
 
     try:
         fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe)
         cleaned_fmp = _normalise_symbols(fmp_symbols)
         if cleaned_fmp:
             await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp")
-            return cleaned_fmp
+            return cleaned_fmp, "fmp"
     except (ProviderError, ValidationError) as exc:
         failures.append(str(exc))
 
@@ -343,7 +385,7 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
             normalised_universe,
             "; ".join(failures[:3]),
         )
-        return cached_symbols
+        return cached_symbols, "cache"
 
     seed_symbols = _normalise_symbols(_SEED_UNIVERSES.get(normalised_universe, []))
     if seed_symbols:
@@ -352,7 +394,7 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
             normalised_universe,
             "; ".join(failures[:3]),
         )
-        return seed_symbols
+        return seed_symbols, "seed"
 
     reason = "; ".join(failures[:6]) if failures else "no provider returned symbols"
     raise ProviderError(f"Universe '{normalised_universe}' returned no valid symbols. Attempts: {reason}")
@@ -418,7 +460,7 @@ async def bootstrap_universe(
     Returns summary counts for added/existing/deleted symbols.
     """
     normalised_universe = _validate_universe(universe)
-    symbols = await fetch_universe_symbols(db, normalised_universe)
+    symbols, source = await fetch_universe_symbols(db, normalised_universe)
 
     existing_rows = await db.execute(select(Ticker.symbol))
     existing_symbols = set(existing_rows.scalars().all())
@@ -446,8 +488,10 @@ async def bootstrap_universe(
 
     return {
         "universe": normalised_universe,
+        "source": source,
         "total_universe_symbols": len(symbols),
         "added": len(symbols_to_add),
         "already_tracked": len(target_symbols & existing_symbols),
         "deleted": deleted_count,
+        "added_symbols": symbols_to_add[:50],
     }