Fix S&P 500 universe parse so renames like BNY are discovered.

Wikipedia no longer uses plain symbol table cells; parse exchange links and NyseSymbol templates, surface the list source in bootstrap results, and keep legacy cell parsing as a fallback.

app/services/ticker_universe_service.py

@@ -55,6 +55,43 @@ if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
 else:
     _CA_BUNDLE_PATH = _CA_BUNDLE
 
+# Wikipedia often returns 403 to non-browser UAs; use a normal browser-like
+# identity for constituent scrapes (no cookies/login).
+_HTTP_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/126.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "en-US,en;q=0.9",
+}
+
+# Modern Wikipedia S&P/Nasdaq tables use exchange templates (NyseSymbol /
+# NasdaqSymbol) rather than a plain <td><a>SYMBOL</a></td>. Prefer quote URLs
+# and template params; keep the legacy cell pattern as a last resort.
+_WIKI_SYMBOL_PATTERNS: tuple[re.Pattern[str], ...] = (
+    re.compile(r"nyse\.com/quote/XNYS:([A-Za-z0-9.-]{1,10})", re.IGNORECASE),
+    re.compile(
+        r"nasdaq\.com/market-activity/stocks/([A-Za-z0-9.-]{1,10})",
+        re.IGNORECASE,
+    ),
+    # {{NyseSymbol|BNY}} / {{NasdaqSymbol|AAPL}} rendered data-mw params
+    re.compile(
+        r'"target":\{"wt":"(?:Nyse|Nasdaq)Symbol"[^}]*\}.*"wt":"([A-Z][A-Z0-9.-]{0,9})"',
+        re.IGNORECASE,
+    ),
+    re.compile(r"<td>\s*<a[^>]*>([A-Z.]{1,10})</a>\s*</td>", re.IGNORECASE),
+)
+
+
+def _extract_wiki_symbols(html: str) -> list[str]:
+    """Pull ticker symbols out of a Wikipedia constituents page."""
+    found: list[str] = []
+    for pattern in _WIKI_SYMBOL_PATTERNS:
+        found.extend(pattern.findall(html))
+    return found
+
 
 def _validate_universe(universe: str) -> str:
     normalised = universe.strip().lower()
@@ -186,20 +223,19 @@ async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]:
     raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'")
 
 
-async def _fetch_html_symbols(
+async def _fetch_wiki_constituent_symbols(
     client: httpx.AsyncClient,
     url: str,
-    pattern: str,
 ) -> tuple[list[str], str | None]:
     try:
-        response = await client.get(url)
+        response = await client.get(url, headers=_HTTP_HEADERS)
     except httpx.HTTPError as exc:
         return [], f"{url}: network error ({type(exc).__name__}: {exc})"
 
     if response.status_code != 200:
         return [], f"{url}: HTTP {response.status_code}"
 
-    matches = re.findall(pattern, response.text, flags=re.IGNORECASE)
+    matches = _extract_wiki_symbols(response.text)
     if not matches:
         return [], f"{url}: no symbols parsed"
     return list(matches), None
@@ -210,7 +246,7 @@ async def _fetch_nasdaq_trader_symbols(
 ) -> tuple[list[str], str | None]:
     url = "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt"
     try:
-        response = await client.get(url)
+        response = await client.get(url, headers=_HTTP_HEADERS)
     except httpx.HTTPError as exc:
         return [], f"{url}: network error ({type(exc).__name__}: {exc})"
 
@@ -240,18 +276,17 @@ async def _fetch_universe_symbols_from_public(universe: str) -> tuple[list[str],
 
     sp500_url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
     nasdaq100_url = "https://en.wikipedia.org/wiki/Nasdaq-100"
-    wiki_symbol_pattern = r"<td>\s*<a[^>]*>([A-Z.]{1,10})</a>\s*</td>"
 
     async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
         if universe == "sp500":
-            symbols, error = await _fetch_html_symbols(client, sp500_url, wiki_symbol_pattern)
+            symbols, error = await _fetch_wiki_constituent_symbols(client, sp500_url)
             if error:
                 failures.append(error)
             else:
                 return symbols, failures, "wikipedia_sp500"
 
         if universe == "nasdaq100":
-            symbols, error = await _fetch_html_symbols(client, nasdaq100_url, wiki_symbol_pattern)
+            symbols, error = await _fetch_wiki_constituent_symbols(client, nasdaq100_url)
             if error:
                 failures.append(error)
             else:
@@ -308,7 +343,10 @@ async def _write_cached_symbols(
     await db.commit()
 
 
-async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
+async def fetch_universe_symbols(
+    db: AsyncSession,
+    universe: str,
+) -> tuple[list[str], str]:
     """Fetch and normalise symbols for a supported universe with fallbacks.
 
     Fallback order:
@@ -316,6 +354,10 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
     2) FMP endpoints (if available)
     3) Cached snapshot in SystemSetting
     4) Built-in seed symbols
+
+    Returns ``(symbols, source_label)`` so bootstrap UI can show where the
+    list came from (important when Wikipedia/FMP fail and a stale cache still
+    lists BK instead of BNY).
     """
     normalised_universe = _validate_universe(universe)
     failures: list[str] = []
@@ -325,14 +367,14 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
     cleaned_public = _normalise_symbols(public_symbols)
     if cleaned_public:
         await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
-        return cleaned_public
+        return cleaned_public, public_source or "public"
 
     try:
         fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe)
         cleaned_fmp = _normalise_symbols(fmp_symbols)
         if cleaned_fmp:
             await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp")
-            return cleaned_fmp
+            return cleaned_fmp, "fmp"
     except (ProviderError, ValidationError) as exc:
         failures.append(str(exc))
 
@@ -343,7 +385,7 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
             normalised_universe,
             "; ".join(failures[:3]),
         )
-        return cached_symbols
+        return cached_symbols, "cache"
 
     seed_symbols = _normalise_symbols(_SEED_UNIVERSES.get(normalised_universe, []))
     if seed_symbols:
@@ -352,7 +394,7 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
             normalised_universe,
             "; ".join(failures[:3]),
         )
-        return seed_symbols
+        return seed_symbols, "seed"
 
     reason = "; ".join(failures[:6]) if failures else "no provider returned symbols"
     raise ProviderError(f"Universe '{normalised_universe}' returned no valid symbols. Attempts: {reason}")
@@ -418,7 +460,7 @@ async def bootstrap_universe(
     Returns summary counts for added/existing/deleted symbols.
     """
     normalised_universe = _validate_universe(universe)
-    symbols = await fetch_universe_symbols(db, normalised_universe)
+    symbols, source = await fetch_universe_symbols(db, normalised_universe)
 
     existing_rows = await db.execute(select(Ticker.symbol))
     existing_symbols = set(existing_rows.scalars().all())
@@ -446,8 +488,10 @@ async def bootstrap_universe(
 
     return {
         "universe": normalised_universe,
+        "source": source,
         "total_universe_symbols": len(symbols),
         "added": len(symbols_to_add),
         "already_tracked": len(target_symbols & existing_symbols),
         "deleted": deleted_count,
+        "added_symbols": symbols_to_add[:50],
     }

frontend/src/hooks/useAdmin.ts

@@ -274,7 +274,10 @@ export function useBootstrapTickers() {
       qc.invalidateQueries({ queryKey: ['admin', 'ticker-universe'] });
       addToast(
         'success',
-        `Bootstrap done: +${result.added}, existing ${result.already_tracked}, deleted ${result.deleted}`,
+        `Bootstrap done (${result.source ?? 'unknown'}): +${result.added}, existing ${result.already_tracked}, deleted ${result.deleted}`
+        + (result.added_symbols?.length
+          ? ` · added ${result.added_symbols.slice(0, 8).join(', ')}${result.added > 8 ? '…' : ''}`
+          : ''),
       );
     },
     onError: (error: Error) => {

frontend/src/lib/types.ts

@@ -718,10 +718,14 @@ export interface TickerUniverseSetting {
 
 export interface TickerUniverseBootstrapResult {
   universe: TickerUniverse;
+  /** Where the member list came from: wikipedia_sp500 | fmp | cache | seed | … */
+  source?: string;
   total_universe_symbols: number;
   added: number;
   already_tracked: number;
   deleted: number;
+  /** Sample of newly added symbols (capped server-side). */
+  added_symbols?: string[];
 }
 
 export interface PipelineReadiness {

tests/unit/test_ticker_universe_service.py

@@ -14,6 +14,7 @@ from app.exceptions import ProviderError
 from app.models.settings import SystemSetting
 from app.models.ticker import Ticker
 from app.services import ticker_universe_service
+from app.services.ticker_universe_service import _extract_wiki_symbols, _normalise_symbols
 
 _engine = create_async_engine("sqlite+aiosqlite://", echo=False)
 _session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
@@ -39,8 +40,8 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo
     session.add(Ticker(symbol="AAPL"))
     await session.commit()
 
-    async def _fake_fetch(_db: AsyncSession, _universe: str) -> list[str]:
-        return ["AAPL", "MSFT", "NVDA"]
+    async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]:
+        return ["AAPL", "MSFT", "NVDA"], "test"
 
     monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch)
 
@@ -49,6 +50,8 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo
     assert result["added"] == 2
     assert result["already_tracked"] == 1
     assert result["deleted"] == 0
+    assert result["source"] == "test"
+    assert set(result["added_symbols"]) == {"MSFT", "NVDA"}
 
     rows = await session.execute(select(Ticker.symbol).order_by(Ticker.symbol.asc()))
     assert list(rows.scalars().all()) == ["AAPL", "MSFT", "NVDA"]
@@ -59,8 +62,8 @@ async def test_bootstrap_universe_prunes_missing_symbols(session: AsyncSession,
     session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="MSFT"), Ticker(symbol="TSLA")])
     await session.commit()
 
-    async def _fake_fetch(_db: AsyncSession, _universe: str) -> list[str]:
-        return ["AAPL", "MSFT"]
+    async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]:
+        return ["AAPL", "MSFT"], "test"
 
     monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch)
 
@@ -100,8 +103,9 @@ async def test_fetch_universe_symbols_uses_cached_snapshot_when_live_sources_fai
     monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
     monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
 
-    symbols = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
+    symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
     assert symbols == ["AAPL", "MSFT"]
+    assert source == "cache"
 
 
 @pytest.mark.asyncio
@@ -118,6 +122,34 @@ async def test_fetch_universe_symbols_uses_seed_when_live_and_cache_fail(
     monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
     monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
 
-    symbols = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
+    symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
     assert "AAPL" in symbols
     assert len(symbols) > 10
+    assert source == "seed"
+
+
+# Snippet shaped like current Wikipedia NyseSymbol / exchange link markup.
+_SAMPLE_WIKI_HTML = """
+<td><a class="external text"
+  data-mw='{"parts":[{"template":{"target":{"wt":"NyseSymbol","href":"./Template:NyseSymbol"},"params":{"1":{"wt":"BNY"}},"i":0}}]}'
+  >BNY</a></td>
+<td><a href="https://www.nyse.com/quote/XNYS:DVN">DVN</a></td>
+<td><a href="https://www.nasdaq.com/market-activity/stocks/aapl">AAPL</a></td>
+<td><a href="https://www.nyse.com/quote/XNYS:MMM">MMM</a></td>
+"""
+
+
+def test_extract_wiki_symbols_finds_bny_and_exchange_links():
+    raw = _extract_wiki_symbols(_SAMPLE_WIKI_HTML)
+    symbols = set(_normalise_symbols(raw))
+    assert "BNY" in symbols
+    assert "DVN" in symbols
+    assert "AAPL" in symbols
+    assert "MMM" in symbols
+    assert "BK" not in symbols
+
+
+def test_legacy_td_anchor_still_works():
+    html = '<tr><td><a href="/wiki/foo">BK</a></td></tr>'
+    symbols = set(_normalise_symbols(_extract_wiki_symbols(html)))
+    assert "BK" in symbols