diff --git a/app/services/ticker_universe_service.py b/app/services/ticker_universe_service.py
index 10659b0..fa1a0c9 100644
--- a/app/services/ticker_universe_service.py
+++ b/app/services/ticker_universe_service.py
@@ -55,6 +55,43 @@ if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
else:
_CA_BUNDLE_PATH = _CA_BUNDLE
+# Wikipedia often returns 403 to non-browser UAs; use a normal browser-like
+# identity for constituent scrapes (no cookies/login).
+_HTTP_HEADERS = {
+ "User-Agent": (
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+ "AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/126.0.0.0 Safari/537.36"
+ ),
+ "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
+ "Accept-Language": "en-US,en;q=0.9",
+}
+
+# Modern Wikipedia S&P/Nasdaq tables use exchange templates (NyseSymbol /
+# NasdaqSymbol) rather than a plain
SYMBOL | . Prefer quote URLs
+# and template params; keep the legacy cell pattern as a last resort.
+_WIKI_SYMBOL_PATTERNS: tuple[re.Pattern[str], ...] = (
+ re.compile(r"nyse\.com/quote/XNYS:([A-Za-z0-9.-]{1,10})", re.IGNORECASE),
+ re.compile(
+ r"nasdaq\.com/market-activity/stocks/([A-Za-z0-9.-]{1,10})",
+ re.IGNORECASE,
+ ),
+ # {{NyseSymbol|BNY}} / {{NasdaqSymbol|AAPL}} rendered data-mw params
+ re.compile(
+ r'"target":\{"wt":"(?:Nyse|Nasdaq)Symbol"[^}]*\}.*"wt":"([A-Z][A-Z0-9.-]{0,9})"',
+ re.IGNORECASE,
+ ),
+ re.compile(r"\s*]*>([A-Z.]{1,10})\s* | ", re.IGNORECASE),
+)
+
+
+def _extract_wiki_symbols(html: str) -> list[str]:
+ """Pull ticker symbols out of a Wikipedia constituents page."""
+ found: list[str] = []
+ for pattern in _WIKI_SYMBOL_PATTERNS:
+ found.extend(pattern.findall(html))
+ return found
+
def _validate_universe(universe: str) -> str:
normalised = universe.strip().lower()
@@ -186,20 +223,19 @@ async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]:
raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'")
-async def _fetch_html_symbols(
+async def _fetch_wiki_constituent_symbols(
client: httpx.AsyncClient,
url: str,
- pattern: str,
) -> tuple[list[str], str | None]:
try:
- response = await client.get(url)
+ response = await client.get(url, headers=_HTTP_HEADERS)
except httpx.HTTPError as exc:
return [], f"{url}: network error ({type(exc).__name__}: {exc})"
if response.status_code != 200:
return [], f"{url}: HTTP {response.status_code}"
- matches = re.findall(pattern, response.text, flags=re.IGNORECASE)
+ matches = _extract_wiki_symbols(response.text)
if not matches:
return [], f"{url}: no symbols parsed"
return list(matches), None
@@ -210,7 +246,7 @@ async def _fetch_nasdaq_trader_symbols(
) -> tuple[list[str], str | None]:
url = "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt"
try:
- response = await client.get(url)
+ response = await client.get(url, headers=_HTTP_HEADERS)
except httpx.HTTPError as exc:
return [], f"{url}: network error ({type(exc).__name__}: {exc})"
@@ -240,18 +276,17 @@ async def _fetch_universe_symbols_from_public(universe: str) -> tuple[list[str],
sp500_url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies"
nasdaq100_url = "https://en.wikipedia.org/wiki/Nasdaq-100"
- wiki_symbol_pattern = r"\s*]*>([A-Z.]{1,10})\s* | "
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
if universe == "sp500":
- symbols, error = await _fetch_html_symbols(client, sp500_url, wiki_symbol_pattern)
+ symbols, error = await _fetch_wiki_constituent_symbols(client, sp500_url)
if error:
failures.append(error)
else:
return symbols, failures, "wikipedia_sp500"
if universe == "nasdaq100":
- symbols, error = await _fetch_html_symbols(client, nasdaq100_url, wiki_symbol_pattern)
+ symbols, error = await _fetch_wiki_constituent_symbols(client, nasdaq100_url)
if error:
failures.append(error)
else:
@@ -308,7 +343,10 @@ async def _write_cached_symbols(
await db.commit()
-async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
+async def fetch_universe_symbols(
+ db: AsyncSession,
+ universe: str,
+) -> tuple[list[str], str]:
"""Fetch and normalise symbols for a supported universe with fallbacks.
Fallback order:
@@ -316,6 +354,10 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
2) FMP endpoints (if available)
3) Cached snapshot in SystemSetting
4) Built-in seed symbols
+
+ Returns ``(symbols, source_label)`` so bootstrap UI can show where the
+ list came from (important when Wikipedia/FMP fail and a stale cache still
+ lists BK instead of BNY).
"""
normalised_universe = _validate_universe(universe)
failures: list[str] = []
@@ -325,14 +367,14 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
cleaned_public = _normalise_symbols(public_symbols)
if cleaned_public:
await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
- return cleaned_public
+ return cleaned_public, public_source or "public"
try:
fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe)
cleaned_fmp = _normalise_symbols(fmp_symbols)
if cleaned_fmp:
await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp")
- return cleaned_fmp
+ return cleaned_fmp, "fmp"
except (ProviderError, ValidationError) as exc:
failures.append(str(exc))
@@ -343,7 +385,7 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
normalised_universe,
"; ".join(failures[:3]),
)
- return cached_symbols
+ return cached_symbols, "cache"
seed_symbols = _normalise_symbols(_SEED_UNIVERSES.get(normalised_universe, []))
if seed_symbols:
@@ -352,7 +394,7 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]:
normalised_universe,
"; ".join(failures[:3]),
)
- return seed_symbols
+ return seed_symbols, "seed"
reason = "; ".join(failures[:6]) if failures else "no provider returned symbols"
raise ProviderError(f"Universe '{normalised_universe}' returned no valid symbols. Attempts: {reason}")
@@ -418,7 +460,7 @@ async def bootstrap_universe(
Returns summary counts for added/existing/deleted symbols.
"""
normalised_universe = _validate_universe(universe)
- symbols = await fetch_universe_symbols(db, normalised_universe)
+ symbols, source = await fetch_universe_symbols(db, normalised_universe)
existing_rows = await db.execute(select(Ticker.symbol))
existing_symbols = set(existing_rows.scalars().all())
@@ -446,8 +488,10 @@ async def bootstrap_universe(
return {
"universe": normalised_universe,
+ "source": source,
"total_universe_symbols": len(symbols),
"added": len(symbols_to_add),
"already_tracked": len(target_symbols & existing_symbols),
"deleted": deleted_count,
+ "added_symbols": symbols_to_add[:50],
}
diff --git a/frontend/src/hooks/useAdmin.ts b/frontend/src/hooks/useAdmin.ts
index 10ae371..d59c6ff 100644
--- a/frontend/src/hooks/useAdmin.ts
+++ b/frontend/src/hooks/useAdmin.ts
@@ -274,7 +274,10 @@ export function useBootstrapTickers() {
qc.invalidateQueries({ queryKey: ['admin', 'ticker-universe'] });
addToast(
'success',
- `Bootstrap done: +${result.added}, existing ${result.already_tracked}, deleted ${result.deleted}`,
+ `Bootstrap done (${result.source ?? 'unknown'}): +${result.added}, existing ${result.already_tracked}, deleted ${result.deleted}`
+ + (result.added_symbols?.length
+ ? ` · added ${result.added_symbols.slice(0, 8).join(', ')}${result.added > 8 ? '…' : ''}`
+ : ''),
);
},
onError: (error: Error) => {
diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts
index e3bbf2e..e8d017f 100644
--- a/frontend/src/lib/types.ts
+++ b/frontend/src/lib/types.ts
@@ -718,10 +718,14 @@ export interface TickerUniverseSetting {
export interface TickerUniverseBootstrapResult {
universe: TickerUniverse;
+ /** Where the member list came from: wikipedia_sp500 | fmp | cache | seed | … */
+ source?: string;
total_universe_symbols: number;
added: number;
already_tracked: number;
deleted: number;
+ /** Sample of newly added symbols (capped server-side). */
+ added_symbols?: string[];
}
export interface PipelineReadiness {
diff --git a/tests/unit/test_ticker_universe_service.py b/tests/unit/test_ticker_universe_service.py
index b6009ce..b17d216 100644
--- a/tests/unit/test_ticker_universe_service.py
+++ b/tests/unit/test_ticker_universe_service.py
@@ -14,6 +14,7 @@ from app.exceptions import ProviderError
from app.models.settings import SystemSetting
from app.models.ticker import Ticker
from app.services import ticker_universe_service
+from app.services.ticker_universe_service import _extract_wiki_symbols, _normalise_symbols
_engine = create_async_engine("sqlite+aiosqlite://", echo=False)
_session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False)
@@ -39,8 +40,8 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo
session.add(Ticker(symbol="AAPL"))
await session.commit()
- async def _fake_fetch(_db: AsyncSession, _universe: str) -> list[str]:
- return ["AAPL", "MSFT", "NVDA"]
+ async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]:
+ return ["AAPL", "MSFT", "NVDA"], "test"
monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch)
@@ -49,6 +50,8 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo
assert result["added"] == 2
assert result["already_tracked"] == 1
assert result["deleted"] == 0
+ assert result["source"] == "test"
+ assert set(result["added_symbols"]) == {"MSFT", "NVDA"}
rows = await session.execute(select(Ticker.symbol).order_by(Ticker.symbol.asc()))
assert list(rows.scalars().all()) == ["AAPL", "MSFT", "NVDA"]
@@ -59,8 +62,8 @@ async def test_bootstrap_universe_prunes_missing_symbols(session: AsyncSession,
session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="MSFT"), Ticker(symbol="TSLA")])
await session.commit()
- async def _fake_fetch(_db: AsyncSession, _universe: str) -> list[str]:
- return ["AAPL", "MSFT"]
+ async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]:
+ return ["AAPL", "MSFT"], "test"
monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch)
@@ -100,8 +103,9 @@ async def test_fetch_universe_symbols_uses_cached_snapshot_when_live_sources_fai
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
- symbols = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
+ symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
assert symbols == ["AAPL", "MSFT"]
+ assert source == "cache"
@pytest.mark.asyncio
@@ -118,6 +122,34 @@ async def test_fetch_universe_symbols_uses_seed_when_live_and_cache_fail(
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public)
monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp)
- symbols = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
+ symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500")
assert "AAPL" in symbols
assert len(symbols) > 10
+ assert source == "seed"
+
+
+# Snippet shaped like current Wikipedia NyseSymbol / exchange link markup.
+_SAMPLE_WIKI_HTML = """
+BNY |
+DVN |
+AAPL |
+MMM |
+"""
+
+
+def test_extract_wiki_symbols_finds_bny_and_exchange_links():
+ raw = _extract_wiki_symbols(_SAMPLE_WIKI_HTML)
+ symbols = set(_normalise_symbols(raw))
+ assert "BNY" in symbols
+ assert "DVN" in symbols
+ assert "AAPL" in symbols
+ assert "MMM" in symbols
+ assert "BK" not in symbols
+
+
+def test_legacy_td_anchor_still_works():
+ html = '| BK |
'
+ symbols = set(_normalise_symbols(_extract_wiki_symbols(html)))
+ assert "BK" in symbols