Fix S&P 500 universe parse so renames like BNY are discovered.

Wikipedia no longer uses plain symbol table cells; parse exchange links and NyseSymbol templates, surface the list source in bootstrap results, and keep legacy cell parsing as a fallback.

tests/unit/test_ticker_universe_service.py

@@ -14,6 +14,7 @@ from app.exceptions import ProviderError
 from app.models.settings import SystemSetting
 from app.models.ticker import Ticker
 from app.services import ticker_universe_service
+from app.services.ticker_universe_service import _extract_wiki_symbols, _normalise_symbols
 
 _engine = create_async_engine("sqlite+aiosqlite://", echo=False)
 _session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
@@ -39,8 +40,8 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo
     session.add(Ticker(symbol="AAPL"))
     await session.commit()
 
-    async def _fake_fetch(_db: AsyncSession, _universe: str) -> list[str]:
-        return ["AAPL", "MSFT", "NVDA"]
+    async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]:
+        return ["AAPL", "MSFT", "NVDA"], "test"
 
     monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch)
 
@@ -49,6 +50,8 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo
     assert result["added"] == 2
     assert result["already_tracked"] == 1
     assert result["deleted"] == 0
+    assert result["source"] == "test"
+    assert set(result["added_symbols"]) == {"MSFT", "NVDA"}
 
     rows = await session.execute(select(Ticker.symbol).order_by(Ticker.symbol.asc()))
     assert list(rows.scalars().all()) == ["AAPL", "MSFT", "NVDA"]
@@ -59,8 +62,8 @@ async def test_bootstrap_universe_prunes_missing_symbols(session: AsyncSession,
     session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="MSFT"), Ticker(symbol="TSLA")])
     await session.commit()
 
-    async def _fake_fetch(_db: AsyncSession, _universe: str) -> list[str]:
-        return ["AAPL", "MSFT"]
+    async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]:
+        return ["AAPL", "MSFT"], "test"
 
     monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch)
 
@@ -100,8 +103,9 @@ async def test_fetch_universe_symbols_uses_cached_snapshot_when_live_sources_fai
     monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
     monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
 
-    symbols = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
+    symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
     assert symbols == ["AAPL", "MSFT"]
+    assert source == "cache"
 
 
 @pytest.mark.asyncio
@@ -118,6 +122,34 @@ async def test_fetch_universe_symbols_uses_seed_when_live_and_cache_fail(
     monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
     monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
 
-    symbols = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
+    symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
     assert "AAPL" in symbols
     assert len(symbols) > 10
+    assert source == "seed"
+
+
+# Snippet shaped like current Wikipedia NyseSymbol / exchange link markup.
+_SAMPLE_WIKI_HTML = """
+<td><a class="external text"
+  data-mw='{"parts":[{"template":{"target":{"wt":"NyseSymbol","href":"./Template:NyseSymbol"},"params":{"1":{"wt":"BNY"}},"i":0}}]}'
+  >BNY</a></td>
+<td><a href="https://www.nyse.com/quote/XNYS:DVN">DVN</a></td>
+<td><a href="https://www.nasdaq.com/market-activity/stocks/aapl">AAPL</a></td>
+<td><a href="https://www.nyse.com/quote/XNYS:MMM">MMM</a></td>
+"""
+
+
+def test_extract_wiki_symbols_finds_bny_and_exchange_links():
+    raw = _extract_wiki_symbols(_SAMPLE_WIKI_HTML)
+    symbols = set(_normalise_symbols(raw))
+    assert "BNY" in symbols
+    assert "DVN" in symbols
+    assert "AAPL" in symbols
+    assert "MMM" in symbols
+    assert "BK" not in symbols
+
+
+def test_legacy_td_anchor_still_works():
+    html = '<tr><td><a href="/wiki/foo">BK</a></td></tr>'
+    symbols = set(_normalise_symbols(_extract_wiki_symbols(html)))
+    assert "BK" in symbols