diff --git a/app/services/ticker_universe_service.py b/app/services/ticker_universe_service.py index 10659b0..fa1a0c9 100644 --- a/app/services/ticker_universe_service.py +++ b/app/services/ticker_universe_service.py @@ -55,6 +55,43 @@ if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists(): else: _CA_BUNDLE_PATH = _CA_BUNDLE +# Wikipedia often returns 403 to non-browser UAs; use a normal browser-like +# identity for constituent scrapes (no cookies/login). +_HTTP_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/126.0.0.0 Safari/537.36" + ), + "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8", + "Accept-Language": "en-US,en;q=0.9", +} + +# Modern Wikipedia S&P/Nasdaq tables use exchange templates (NyseSymbol / +# NasdaqSymbol) rather than a plain SYMBOL. Prefer quote URLs +# and template params; keep the legacy cell pattern as a last resort. +_WIKI_SYMBOL_PATTERNS: tuple[re.Pattern[str], ...] = ( + re.compile(r"nyse\.com/quote/XNYS:([A-Za-z0-9.-]{1,10})", re.IGNORECASE), + re.compile( + r"nasdaq\.com/market-activity/stocks/([A-Za-z0-9.-]{1,10})", + re.IGNORECASE, + ), + # {{NyseSymbol|BNY}} / {{NasdaqSymbol|AAPL}} rendered data-mw params + re.compile( + r'"target":\{"wt":"(?:Nyse|Nasdaq)Symbol"[^}]*\}.*"wt":"([A-Z][A-Z0-9.-]{0,9})"', + re.IGNORECASE, + ), + re.compile(r"\s*]*>([A-Z.]{1,10})\s*", re.IGNORECASE), +) + + +def _extract_wiki_symbols(html: str) -> list[str]: + """Pull ticker symbols out of a Wikipedia constituents page.""" + found: list[str] = [] + for pattern in _WIKI_SYMBOL_PATTERNS: + found.extend(pattern.findall(html)) + return found + def _validate_universe(universe: str) -> str: normalised = universe.strip().lower() @@ -186,20 +223,19 @@ async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]: raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'") -async def _fetch_html_symbols( +async def _fetch_wiki_constituent_symbols( client: httpx.AsyncClient, url: str, - pattern: str, ) -> tuple[list[str], str | None]: try: - response = await client.get(url) + response = await client.get(url, headers=_HTTP_HEADERS) except httpx.HTTPError as exc: return [], f"{url}: network error ({type(exc).__name__}: {exc})" if response.status_code != 200: return [], f"{url}: HTTP {response.status_code}" - matches = re.findall(pattern, response.text, flags=re.IGNORECASE) + matches = _extract_wiki_symbols(response.text) if not matches: return [], f"{url}: no symbols parsed" return list(matches), None @@ -210,7 +246,7 @@ async def _fetch_nasdaq_trader_symbols( ) -> tuple[list[str], str | None]: url = "https://www.nasdaqtrader.com/dynamic/SymDir/nasdaqlisted.txt" try: - response = await client.get(url) + response = await client.get(url, headers=_HTTP_HEADERS) except httpx.HTTPError as exc: return [], f"{url}: network error ({type(exc).__name__}: {exc})" @@ -240,18 +276,17 @@ async def _fetch_universe_symbols_from_public(universe: str) -> tuple[list[str], sp500_url = "https://en.wikipedia.org/wiki/List_of_S%26P_500_companies" nasdaq100_url = "https://en.wikipedia.org/wiki/Nasdaq-100" - wiki_symbol_pattern = r"\s*]*>([A-Z.]{1,10})\s*" async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client: if universe == "sp500": - symbols, error = await _fetch_html_symbols(client, sp500_url, wiki_symbol_pattern) + symbols, error = await _fetch_wiki_constituent_symbols(client, sp500_url) if error: failures.append(error) else: return symbols, failures, "wikipedia_sp500" if universe == "nasdaq100": - symbols, error = await _fetch_html_symbols(client, nasdaq100_url, wiki_symbol_pattern) + symbols, error = await _fetch_wiki_constituent_symbols(client, nasdaq100_url) if error: failures.append(error) else: @@ -308,7 +343,10 @@ async def _write_cached_symbols( await db.commit() -async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]: +async def fetch_universe_symbols( + db: AsyncSession, + universe: str, +) -> tuple[list[str], str]: """Fetch and normalise symbols for a supported universe with fallbacks. Fallback order: @@ -316,6 +354,10 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]: 2) FMP endpoints (if available) 3) Cached snapshot in SystemSetting 4) Built-in seed symbols + + Returns ``(symbols, source_label)`` so bootstrap UI can show where the + list came from (important when Wikipedia/FMP fail and a stale cache still + lists BK instead of BNY). """ normalised_universe = _validate_universe(universe) failures: list[str] = [] @@ -325,14 +367,14 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]: cleaned_public = _normalise_symbols(public_symbols) if cleaned_public: await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public") - return cleaned_public + return cleaned_public, public_source or "public" try: fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe) cleaned_fmp = _normalise_symbols(fmp_symbols) if cleaned_fmp: await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp") - return cleaned_fmp + return cleaned_fmp, "fmp" except (ProviderError, ValidationError) as exc: failures.append(str(exc)) @@ -343,7 +385,7 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]: normalised_universe, "; ".join(failures[:3]), ) - return cached_symbols + return cached_symbols, "cache" seed_symbols = _normalise_symbols(_SEED_UNIVERSES.get(normalised_universe, [])) if seed_symbols: @@ -352,7 +394,7 @@ async def fetch_universe_symbols(db: AsyncSession, universe: str) -> list[str]: normalised_universe, "; ".join(failures[:3]), ) - return seed_symbols + return seed_symbols, "seed" reason = "; ".join(failures[:6]) if failures else "no provider returned symbols" raise ProviderError(f"Universe '{normalised_universe}' returned no valid symbols. Attempts: {reason}") @@ -418,7 +460,7 @@ async def bootstrap_universe( Returns summary counts for added/existing/deleted symbols. """ normalised_universe = _validate_universe(universe) - symbols = await fetch_universe_symbols(db, normalised_universe) + symbols, source = await fetch_universe_symbols(db, normalised_universe) existing_rows = await db.execute(select(Ticker.symbol)) existing_symbols = set(existing_rows.scalars().all()) @@ -446,8 +488,10 @@ async def bootstrap_universe( return { "universe": normalised_universe, + "source": source, "total_universe_symbols": len(symbols), "added": len(symbols_to_add), "already_tracked": len(target_symbols & existing_symbols), "deleted": deleted_count, + "added_symbols": symbols_to_add[:50], } diff --git a/frontend/src/hooks/useAdmin.ts b/frontend/src/hooks/useAdmin.ts index 10ae371..d59c6ff 100644 --- a/frontend/src/hooks/useAdmin.ts +++ b/frontend/src/hooks/useAdmin.ts @@ -274,7 +274,10 @@ export function useBootstrapTickers() { qc.invalidateQueries({ queryKey: ['admin', 'ticker-universe'] }); addToast( 'success', - `Bootstrap done: +${result.added}, existing ${result.already_tracked}, deleted ${result.deleted}`, + `Bootstrap done (${result.source ?? 'unknown'}): +${result.added}, existing ${result.already_tracked}, deleted ${result.deleted}` + + (result.added_symbols?.length + ? ` · added ${result.added_symbols.slice(0, 8).join(', ')}${result.added > 8 ? '…' : ''}` + : ''), ); }, onError: (error: Error) => { diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index e3bbf2e..e8d017f 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -718,10 +718,14 @@ export interface TickerUniverseSetting { export interface TickerUniverseBootstrapResult { universe: TickerUniverse; + /** Where the member list came from: wikipedia_sp500 | fmp | cache | seed | … */ + source?: string; total_universe_symbols: number; added: number; already_tracked: number; deleted: number; + /** Sample of newly added symbols (capped server-side). */ + added_symbols?: string[]; } export interface PipelineReadiness { diff --git a/tests/unit/test_ticker_universe_service.py b/tests/unit/test_ticker_universe_service.py index b6009ce..b17d216 100644 --- a/tests/unit/test_ticker_universe_service.py +++ b/tests/unit/test_ticker_universe_service.py @@ -14,6 +14,7 @@ from app.exceptions import ProviderError from app.models.settings import SystemSetting from app.models.ticker import Ticker from app.services import ticker_universe_service +from app.services.ticker_universe_service import _extract_wiki_symbols, _normalise_symbols _engine = create_async_engine("sqlite+aiosqlite://", echo=False) _session_factory = async_sessionmaker(_engine, class_=AsyncSession, expire_on_commit=False) @@ -39,8 +40,8 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo session.add(Ticker(symbol="AAPL")) await session.commit() - async def _fake_fetch(_db: AsyncSession, _universe: str) -> list[str]: - return ["AAPL", "MSFT", "NVDA"] + async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]: + return ["AAPL", "MSFT", "NVDA"], "test" monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch) @@ -49,6 +50,8 @@ async def test_bootstrap_universe_adds_missing_symbols(session: AsyncSession, mo assert result["added"] == 2 assert result["already_tracked"] == 1 assert result["deleted"] == 0 + assert result["source"] == "test" + assert set(result["added_symbols"]) == {"MSFT", "NVDA"} rows = await session.execute(select(Ticker.symbol).order_by(Ticker.symbol.asc())) assert list(rows.scalars().all()) == ["AAPL", "MSFT", "NVDA"] @@ -59,8 +62,8 @@ async def test_bootstrap_universe_prunes_missing_symbols(session: AsyncSession, session.add_all([Ticker(symbol="AAPL"), Ticker(symbol="MSFT"), Ticker(symbol="TSLA")]) await session.commit() - async def _fake_fetch(_db: AsyncSession, _universe: str) -> list[str]: - return ["AAPL", "MSFT"] + async def _fake_fetch(_db: AsyncSession, _universe: str) -> tuple[list[str], str]: + return ["AAPL", "MSFT"], "test" monkeypatch.setattr(ticker_universe_service, "fetch_universe_symbols", _fake_fetch) @@ -100,8 +103,9 @@ async def test_fetch_universe_symbols_uses_cached_snapshot_when_live_sources_fai monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public) monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp) - symbols = await ticker_universe_service.fetch_universe_symbols(session, "sp500") + symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500") assert symbols == ["AAPL", "MSFT"] + assert source == "cache" @pytest.mark.asyncio @@ -118,6 +122,34 @@ async def test_fetch_universe_symbols_uses_seed_when_live_and_cache_fail( monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public) monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp) - symbols = await ticker_universe_service.fetch_universe_symbols(session, "sp500") + symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500") assert "AAPL" in symbols assert len(symbols) > 10 + assert source == "seed" + + +# Snippet shaped like current Wikipedia NyseSymbol / exchange link markup. +_SAMPLE_WIKI_HTML = """ +BNY +DVN +AAPL +MMM +""" + + +def test_extract_wiki_symbols_finds_bny_and_exchange_links(): + raw = _extract_wiki_symbols(_SAMPLE_WIKI_HTML) + symbols = set(_normalise_symbols(raw)) + assert "BNY" in symbols + assert "DVN" in symbols + assert "AAPL" in symbols + assert "MMM" in symbols + assert "BK" not in symbols + + +def test_legacy_td_anchor_still_works(): + html = 'BK' + symbols = set(_normalise_symbols(_extract_wiki_symbols(html))) + assert "BK" in symbols