chore: decommission FMP, Finnhub and Alpha Vantage (A6)
The A5 cutover has been on and observed in production, so SEC Company Facts + DoltHub earnings are already the live source for `fundamental_data`. This removes everything the legacy path still occupied. Gone: the three providers and their config/env keys; the weekly `fundamental_collector` job; the cutover toggle (SEC + Dolt is now the unconditional path, so `off` can no longer silently freeze scoring inputs); the A5 parity report, whose deltas became structurally zero once the candidate builder started writing the table it compared against; and the FMP tier of universe bootstrap. Two behavioral notes: - Disabling **SEC Fundamentals Import** now stops the SEC network fetch only. The local cache refresh moved outside the job-enable check, because candidates also derive from daily closes and earnings events — freezing those on an ingestion pause would stale scoring with no fallback left to recover from. - `/ingestion/fetch?sources=fundamentals` still accepts the key and reports `skipped`; there is no per-ticker fetch any more. Migration 029 does not blanket-delete the leftover settings rows. Migrations run before the service restart, and pre-A6 code reads an absent `job_*_enabled` row as *enabled* — so the two behavior-bearing keys become tombstones pinned to safe values (hidden in Admin) and only the inert three are deleted. Removing the provider keys from the production `.env` is the matching rollout step. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
@@ -113,116 +113,6 @@ def _normalise_symbols(symbols: Iterable[str]) -> list[str]:
|
||||
return sorted(deduped)
|
||||
|
||||
|
||||
def _extract_symbols_from_fmp_payload(payload: object) -> list[str]:
|
||||
if not isinstance(payload, list):
|
||||
return []
|
||||
|
||||
symbols: list[str] = []
|
||||
for item in payload:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
candidate = item.get("symbol") or item.get("ticker")
|
||||
if isinstance(candidate, str):
|
||||
symbols.append(candidate)
|
||||
return symbols
|
||||
|
||||
|
||||
async def _try_fmp_urls(
|
||||
client: httpx.AsyncClient,
|
||||
urls: list[str],
|
||||
) -> tuple[list[str], list[str]]:
|
||||
failures: list[str] = []
|
||||
for url in urls:
|
||||
endpoint = url.split("?")[0]
|
||||
try:
|
||||
response = await client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
failures.append(f"{endpoint}: network error ({type(exc).__name__}: {exc})")
|
||||
continue
|
||||
|
||||
if response.status_code != 200:
|
||||
failures.append(f"{endpoint}: HTTP {response.status_code}")
|
||||
continue
|
||||
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
failures.append(f"{endpoint}: invalid JSON payload")
|
||||
continue
|
||||
|
||||
symbols = _extract_symbols_from_fmp_payload(payload)
|
||||
if symbols:
|
||||
return symbols, failures
|
||||
|
||||
failures.append(f"{endpoint}: empty/unsupported payload")
|
||||
|
||||
return [], failures
|
||||
|
||||
|
||||
async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]:
|
||||
if not settings.fmp_api_key:
|
||||
raise ValidationError(
|
||||
"FMP API key is required for universe bootstrap (set FMP_API_KEY)"
|
||||
)
|
||||
|
||||
api_key = settings.fmp_api_key
|
||||
stable_base = "https://financialmodelingprep.com/stable"
|
||||
legacy_base = "https://financialmodelingprep.com/api/v3"
|
||||
|
||||
stable_candidates: dict[str, list[str]] = {
|
||||
"sp500": [
|
||||
f"{stable_base}/sp500-constituent?apikey={api_key}",
|
||||
f"{stable_base}/sp500-constituents?apikey={api_key}",
|
||||
],
|
||||
"nasdaq100": [
|
||||
f"{stable_base}/nasdaq-100-constituent?apikey={api_key}",
|
||||
f"{stable_base}/nasdaq100-constituent?apikey={api_key}",
|
||||
f"{stable_base}/nasdaq-100-constituents?apikey={api_key}",
|
||||
],
|
||||
"nasdaq_all": [
|
||||
f"{stable_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
|
||||
f"{stable_base}/available-traded/list?apikey={api_key}",
|
||||
],
|
||||
}
|
||||
|
||||
legacy_candidates: dict[str, list[str]] = {
|
||||
"sp500": [
|
||||
f"{legacy_base}/sp500_constituent?apikey={api_key}",
|
||||
f"{legacy_base}/sp500_constituent",
|
||||
],
|
||||
"nasdaq100": [
|
||||
f"{legacy_base}/nasdaq_constituent?apikey={api_key}",
|
||||
f"{legacy_base}/nasdaq_constituent",
|
||||
],
|
||||
"nasdaq_all": [
|
||||
f"{legacy_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
|
||||
],
|
||||
}
|
||||
|
||||
failures: list[str] = []
|
||||
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
|
||||
stable_symbols, stable_failures = await _try_fmp_urls(client, stable_candidates[universe])
|
||||
failures.extend(stable_failures)
|
||||
|
||||
if stable_symbols:
|
||||
return stable_symbols
|
||||
|
||||
legacy_symbols, legacy_failures = await _try_fmp_urls(client, legacy_candidates[universe])
|
||||
failures.extend(legacy_failures)
|
||||
|
||||
if legacy_symbols:
|
||||
return legacy_symbols
|
||||
|
||||
if failures:
|
||||
reason = "; ".join(failures[:6])
|
||||
logger.warning("FMP universe fetch failed for %s: %s", universe, reason)
|
||||
raise ProviderError(
|
||||
f"Failed to fetch universe symbols from FMP for '{universe}'. Attempts: {reason}"
|
||||
)
|
||||
|
||||
raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'")
|
||||
|
||||
|
||||
async def _fetch_wiki_constituent_symbols(
|
||||
client: httpx.AsyncClient,
|
||||
url: str,
|
||||
@@ -351,13 +241,16 @@ async def fetch_universe_symbols(
|
||||
|
||||
Fallback order:
|
||||
1) Free public sources (Wikipedia/NASDAQ trader)
|
||||
2) FMP endpoints (if available)
|
||||
3) Cached snapshot in SystemSetting
|
||||
4) Built-in seed symbols
|
||||
2) Cached snapshot in SystemSetting
|
||||
3) Built-in seed symbols
|
||||
|
||||
Returns ``(symbols, source_label)`` so bootstrap UI can show where the
|
||||
list came from (important when Wikipedia/FMP fail and a stale cache still
|
||||
lists BK instead of BNY).
|
||||
list came from (important when the public source fails and a stale cache
|
||||
still lists BK instead of BNY).
|
||||
|
||||
The seeds are representative, not complete, so a *fresh* install whose
|
||||
public source is down bootstraps a partial universe. A warm instance is
|
||||
unaffected — it falls through to its cached snapshot.
|
||||
"""
|
||||
normalised_universe = _validate_universe(universe)
|
||||
failures: list[str] = []
|
||||
@@ -369,15 +262,6 @@ async def fetch_universe_symbols(
|
||||
await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
|
||||
return cleaned_public, public_source or "public"
|
||||
|
||||
try:
|
||||
fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe)
|
||||
cleaned_fmp = _normalise_symbols(fmp_symbols)
|
||||
if cleaned_fmp:
|
||||
await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp")
|
||||
return cleaned_fmp, "fmp"
|
||||
except (ProviderError, ValidationError) as exc:
|
||||
failures.append(str(exc))
|
||||
|
||||
cached_symbols = await _read_cached_symbols(db, normalised_universe)
|
||||
if cached_symbols:
|
||||
logger.warning(
|
||||
|
||||
Reference in New Issue
Block a user