chore: decommission FMP, Finnhub and Alpha Vantage (A6)

The A5 cutover has been on and observed in production, so SEC Company Facts +
DoltHub earnings are already the live source for `fundamental_data`. This
removes everything the legacy path still occupied.

Gone: the three providers and their config/env keys; the weekly
`fundamental_collector` job; the cutover toggle (SEC + Dolt is now the
unconditional path, so `off` can no longer silently freeze scoring inputs); the
A5 parity report, whose deltas became structurally zero once the candidate
builder started writing the table it compared against; and the FMP tier of
universe bootstrap.

Two behavioral notes:

- Disabling **SEC Fundamentals Import** now stops the SEC network fetch only.
  The local cache refresh moved outside the job-enable check, because candidates
  also derive from daily closes and earnings events — freezing those on an
  ingestion pause would stale scoring with no fallback left to recover from.
- `/ingestion/fetch?sources=fundamentals` still accepts the key and reports
  `skipped`; there is no per-ticker fetch any more.

Migration 029 does not blanket-delete the leftover settings rows. Migrations run
before the service restart, and pre-A6 code reads an absent `job_*_enabled` row
as *enabled* — so the two behavior-bearing keys become tombstones pinned to safe
values (hidden in Admin) and only the inert three are deleted. Removing the
provider keys from the production `.env` is the matching rollout step.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-07 11:19:28 +02:00
co-authored by Claude Opus 5
parent f5d4b516ab
commit 3e83d63b05
51 changed files with 368 additions and 3787 deletions
+8 -124
View File
@@ -113,116 +113,6 @@ def _normalise_symbols(symbols: Iterable[str]) -> list[str]:
return sorted(deduped)
def _extract_symbols_from_fmp_payload(payload: object) -> list[str]:
if not isinstance(payload, list):
return []
symbols: list[str] = []
for item in payload:
if not isinstance(item, dict):
continue
candidate = item.get("symbol") or item.get("ticker")
if isinstance(candidate, str):
symbols.append(candidate)
return symbols
async def _try_fmp_urls(
client: httpx.AsyncClient,
urls: list[str],
) -> tuple[list[str], list[str]]:
failures: list[str] = []
for url in urls:
endpoint = url.split("?")[0]
try:
response = await client.get(url)
except httpx.HTTPError as exc:
failures.append(f"{endpoint}: network error ({type(exc).__name__}: {exc})")
continue
if response.status_code != 200:
failures.append(f"{endpoint}: HTTP {response.status_code}")
continue
try:
payload = response.json()
except ValueError:
failures.append(f"{endpoint}: invalid JSON payload")
continue
symbols = _extract_symbols_from_fmp_payload(payload)
if symbols:
return symbols, failures
failures.append(f"{endpoint}: empty/unsupported payload")
return [], failures
async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]:
if not settings.fmp_api_key:
raise ValidationError(
"FMP API key is required for universe bootstrap (set FMP_API_KEY)"
)
api_key = settings.fmp_api_key
stable_base = "https://financialmodelingprep.com/stable"
legacy_base = "https://financialmodelingprep.com/api/v3"
stable_candidates: dict[str, list[str]] = {
"sp500": [
f"{stable_base}/sp500-constituent?apikey={api_key}",
f"{stable_base}/sp500-constituents?apikey={api_key}",
],
"nasdaq100": [
f"{stable_base}/nasdaq-100-constituent?apikey={api_key}",
f"{stable_base}/nasdaq100-constituent?apikey={api_key}",
f"{stable_base}/nasdaq-100-constituents?apikey={api_key}",
],
"nasdaq_all": [
f"{stable_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
f"{stable_base}/available-traded/list?apikey={api_key}",
],
}
legacy_candidates: dict[str, list[str]] = {
"sp500": [
f"{legacy_base}/sp500_constituent?apikey={api_key}",
f"{legacy_base}/sp500_constituent",
],
"nasdaq100": [
f"{legacy_base}/nasdaq_constituent?apikey={api_key}",
f"{legacy_base}/nasdaq_constituent",
],
"nasdaq_all": [
f"{legacy_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}",
],
}
failures: list[str] = []
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
stable_symbols, stable_failures = await _try_fmp_urls(client, stable_candidates[universe])
failures.extend(stable_failures)
if stable_symbols:
return stable_symbols
legacy_symbols, legacy_failures = await _try_fmp_urls(client, legacy_candidates[universe])
failures.extend(legacy_failures)
if legacy_symbols:
return legacy_symbols
if failures:
reason = "; ".join(failures[:6])
logger.warning("FMP universe fetch failed for %s: %s", universe, reason)
raise ProviderError(
f"Failed to fetch universe symbols from FMP for '{universe}'. Attempts: {reason}"
)
raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'")
async def _fetch_wiki_constituent_symbols(
client: httpx.AsyncClient,
url: str,
@@ -351,13 +241,16 @@ async def fetch_universe_symbols(
Fallback order:
1) Free public sources (Wikipedia/NASDAQ trader)
2) FMP endpoints (if available)
3) Cached snapshot in SystemSetting
4) Built-in seed symbols
2) Cached snapshot in SystemSetting
3) Built-in seed symbols
Returns ``(symbols, source_label)`` so bootstrap UI can show where the
list came from (important when Wikipedia/FMP fail and a stale cache still
lists BK instead of BNY).
list came from (important when the public source fails and a stale cache
still lists BK instead of BNY).
The seeds are representative, not complete, so a *fresh* install whose
public source is down bootstraps a partial universe. A warm instance is
unaffected — it falls through to its cached snapshot.
"""
normalised_universe = _validate_universe(universe)
failures: list[str] = []
@@ -369,15 +262,6 @@ async def fetch_universe_symbols(
await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public")
return cleaned_public, public_source or "public"
try:
fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe)
cleaned_fmp = _normalise_symbols(fmp_symbols)
if cleaned_fmp:
await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp")
return cleaned_fmp, "fmp"
except (ProviderError, ValidationError) as exc:
failures.append(str(exc))
cached_symbols = await _read_cached_symbols(db, normalised_universe)
if cached_symbols:
logger.warning(