chore: decommission FMP, Finnhub and Alpha Vantage (A6)

The A5 cutover has been on and observed in production, so SEC Company Facts +
DoltHub earnings are already the live source for `fundamental_data`. This
removes everything the legacy path still occupied.

Gone: the three providers and their config/env keys; the weekly
`fundamental_collector` job; the cutover toggle (SEC + Dolt is now the
unconditional path, so `off` can no longer silently freeze scoring inputs); the
A5 parity report, whose deltas became structurally zero once the candidate
builder started writing the table it compared against; and the FMP tier of
universe bootstrap.

Two behavioral notes:

- Disabling **SEC Fundamentals Import** now stops the SEC network fetch only.
  The local cache refresh moved outside the job-enable check, because candidates
  also derive from daily closes and earnings events — freezing those on an
  ingestion pause would stale scoring with no fallback left to recover from.
- `/ingestion/fetch?sources=fundamentals` still accepts the key and reports
  `skipped`; there is no per-ticker fetch any more.

Migration 029 does not blanket-delete the leftover settings rows. Migrations run
before the service restart, and pre-A6 code reads an absent `job_*_enabled` row
as *enabled* — so the two behavior-bearing keys become tombstones pinned to safe
values (hidden in Admin) and only the inert three are deleted. Removing the
provider keys from the production `.env` is the matching rollout step.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-08-07 11:19:28 +02:00
co-authored by Claude Opus 5
parent f5d4b516ab
commit 3e83d63b05
51 changed files with 368 additions and 3787 deletions
-174
View File
@@ -1,174 +0,0 @@
"""Financial Modeling Prep (FMP) fundamentals provider using httpx.
Uses the stable API endpoints (https://financialmodelingprep.com/stable/)
which replaced the legacy /api/v3/ endpoints deprecated in Aug 2025.
"""
from __future__ import annotations
import logging
import os
from datetime import datetime, timezone
from pathlib import Path
import httpx
from app.exceptions import ProviderError, RateLimitError
from app.providers.protocol import FundamentalData
logger = logging.getLogger(__name__)
_FMP_STABLE_URL = "https://financialmodelingprep.com/stable"
# Resolve CA bundle for explicit httpx verify
_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
_CA_BUNDLE_PATH: str | bool = True # use system default
else:
_CA_BUNDLE_PATH = _CA_BUNDLE
class FMPFundamentalProvider:
"""Fetches fundamental data from Financial Modeling Prep REST API."""
def __init__(self, api_key: str) -> None:
if not api_key:
raise ProviderError("FMP API key is required")
self._api_key = api_key
# Mapping from FMP endpoint name to the FundamentalData field it populates
_ENDPOINT_FIELD_MAP: dict[str, str] = {
"ratios-ttm": "pe_ratio",
"financial-growth": "revenue_growth",
"earnings": "earnings_surprise",
}
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
"""Fetch P/E, revenue growth, earnings surprise, and market cap.
Fetches from multiple stable endpoints. If a supplementary endpoint
(ratios, growth, earnings) returns 402 (paid tier), we gracefully
degrade and return partial data rather than failing entirely, and
record the affected field in ``unavailable_fields``.
"""
try:
endpoints_402: set[str] = set()
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
params = {"symbol": ticker, "apikey": self._api_key}
# Profile is the primary source — must succeed
profile = await self._fetch_json(client, "profile", params, ticker)
# Supplementary sources — degrade gracefully on 402
ratios, was_402 = await self._fetch_json_optional(client, "ratios-ttm", params, ticker)
if was_402:
endpoints_402.add("ratios-ttm")
growth, was_402 = await self._fetch_json_optional(client, "financial-growth", params, ticker)
if was_402:
endpoints_402.add("financial-growth")
earnings, was_402 = await self._fetch_json_optional(client, "earnings", params, ticker)
if was_402:
endpoints_402.add("earnings")
pe_ratio = self._safe_float(ratios.get("priceToEarningsRatioTTM"))
revenue_growth = self._safe_float(growth.get("revenueGrowth"))
market_cap = self._safe_float(profile.get("marketCap"))
earnings_surprise = self._compute_earnings_surprise(earnings)
# Build unavailable_fields from 402 endpoints
unavailable_fields: dict[str, str] = {
self._ENDPOINT_FIELD_MAP[ep]: "requires paid plan"
for ep in endpoints_402
if ep in self._ENDPOINT_FIELD_MAP
}
return FundamentalData(
ticker=ticker,
pe_ratio=pe_ratio,
revenue_growth=revenue_growth,
earnings_surprise=earnings_surprise,
market_cap=market_cap,
fetched_at=datetime.now(timezone.utc),
unavailable_fields=unavailable_fields,
)
except (ProviderError, RateLimitError):
raise
except Exception as exc:
logger.error("FMP provider error for %s: %s", ticker, exc)
raise ProviderError(f"FMP provider error for {ticker}: {exc}") from exc
async def _fetch_json(
self,
client: httpx.AsyncClient,
endpoint: str,
params: dict,
ticker: str,
) -> dict:
"""Fetch a stable endpoint and return the first item (or empty dict)."""
url = f"{_FMP_STABLE_URL}/{endpoint}"
resp = await client.get(url, params=params)
self._check_response(resp, ticker, endpoint)
data = resp.json()
if isinstance(data, list):
return data[0] if data else {}
return data if isinstance(data, dict) else {}
async def _fetch_json_optional(
self,
client: httpx.AsyncClient,
endpoint: str,
params: dict,
ticker: str,
) -> tuple[dict, bool]:
"""Fetch a stable endpoint, returning ``({}, True)`` on 402 (paid tier).
Returns a tuple of (data_dict, was_402) so callers can track which
endpoints required a paid plan.
"""
url = f"{_FMP_STABLE_URL}/{endpoint}"
resp = await client.get(url, params=params)
if resp.status_code == 402:
logger.warning("FMP %s requires paid plan — skipping for %s", endpoint, ticker)
return {}, True
self._check_response(resp, ticker, endpoint)
data = resp.json()
if isinstance(data, list):
return (data[0] if data else {}, False)
return (data if isinstance(data, dict) else {}, False)
def _compute_earnings_surprise(self, earnings_data: dict) -> float | None:
"""Compute earnings surprise % from the most recent actual vs estimated EPS."""
actual = self._safe_float(earnings_data.get("epsActual"))
estimated = self._safe_float(earnings_data.get("epsEstimated"))
if actual is None or estimated is None or estimated == 0:
return None
return ((actual - estimated) / abs(estimated)) * 100
def _check_response(
self, resp: httpx.Response, ticker: str, endpoint: str
) -> None:
"""Raise appropriate errors for non-200 responses."""
if resp.status_code == 429:
raise RateLimitError(f"FMP rate limit hit for {ticker} ({endpoint})")
if resp.status_code == 403:
raise ProviderError(
f"FMP {endpoint} access denied for {ticker}: HTTP 403 — check API key validity and plan tier"
)
if resp.status_code != 200:
raise ProviderError(
f"FMP {endpoint} error for {ticker}: HTTP {resp.status_code}"
)
@staticmethod
def _safe_float(value: object) -> float | None:
"""Convert a value to float, returning None on failure."""
if value is None:
return None
try:
return float(value)
except (TypeError, ValueError):
return None
-354
View File
@@ -1,354 +0,0 @@
"""Chained fundamentals provider with fallback adapters.
Order:
1) FMP (if configured)
2) Finnhub (if configured)
3) Alpha Vantage (if configured)
"""
from __future__ import annotations
import logging
import os
from datetime import date, datetime, timedelta, timezone
from pathlib import Path
import httpx
from app.config import settings
from app.exceptions import ProviderError, RateLimitError
from app.providers.fmp import FMPFundamentalProvider
from app.providers.protocol import FundamentalData, FundamentalProvider
logger = logging.getLogger(__name__)
_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "")
if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists():
_CA_BUNDLE_PATH: str | bool = True
else:
_CA_BUNDLE_PATH = _CA_BUNDLE
def _safe_float(value: object) -> float | None:
if value is None:
return None
try:
return float(value)
except (TypeError, ValueError):
return None
def _to_api_symbol(symbol: str) -> str:
"""Convert internal symbol format (BRK-B) to API format (BRK.B).
Finnhub and Alpha Vantage use dot-separated share class notation.
"""
return symbol.replace("-", ".")
class FinnhubFundamentalProvider:
"""Fundamentals provider backed by Finnhub free endpoints."""
def __init__(self, api_key: str) -> None:
if not api_key:
raise ProviderError("Finnhub API key is required")
self._api_key = api_key
self._base_url = "https://finnhub.io/api/v1"
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
unavailable: dict[str, str] = {}
api_symbol = _to_api_symbol(ticker)
today = date.today()
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
profile_resp = await client.get(
f"{self._base_url}/stock/profile2",
params={"symbol": api_symbol, "token": self._api_key},
)
metric_resp = await client.get(
f"{self._base_url}/stock/metric",
params={"symbol": api_symbol, "metric": "all", "token": self._api_key},
)
earnings_resp = await client.get(
f"{self._base_url}/stock/earnings",
params={"symbol": api_symbol, "limit": 1, "token": self._api_key},
)
calendar_resp = await client.get(
f"{self._base_url}/calendar/earnings",
params={
"symbol": api_symbol,
"from": today.isoformat(),
"to": (today + timedelta(days=120)).isoformat(),
"token": self._api_key,
},
)
for resp, endpoint in (
(profile_resp, "profile2"),
(metric_resp, "stock/metric"),
(earnings_resp, "stock/earnings"),
(calendar_resp, "calendar/earnings"),
):
if resp.status_code == 429:
raise RateLimitError(f"Finnhub rate limit hit for {ticker} ({endpoint})")
if resp.status_code in (401, 403):
raise ProviderError(f"Finnhub access denied for {ticker} ({endpoint}): HTTP {resp.status_code}")
if resp.status_code != 200:
raise ProviderError(f"Finnhub error for {ticker} ({endpoint}): HTTP {resp.status_code}")
profile_payload = profile_resp.json() if profile_resp.text else {}
metric_payload = metric_resp.json() if metric_resp.text else {}
earnings_payload = earnings_resp.json() if earnings_resp.text else []
metrics = metric_payload.get("metric", {}) if isinstance(metric_payload, dict) else {}
# Finnhub profile2 marketCapitalization is in millions of USD.
# Normalize to absolute dollars so cap bands / formatters match FMP & Alpha Vantage.
market_cap_millions = _safe_float((profile_payload or {}).get("marketCapitalization"))
market_cap = market_cap_millions * 1_000_000.0 if market_cap_millions is not None else None
pe_ratio = _safe_float(metrics.get("peTTM") or metrics.get("peNormalizedAnnual"))
revenue_growth = _safe_float(metrics.get("revenueGrowthTTMYoy") or metrics.get("revenueGrowth5Y"))
earnings_surprise = None
if isinstance(earnings_payload, list) and earnings_payload:
first = earnings_payload[0] if isinstance(earnings_payload[0], dict) else {}
earnings_surprise = _safe_float(first.get("surprisePercent"))
next_earnings_date = self._next_earnings(calendar_resp)
if pe_ratio is None:
unavailable["pe_ratio"] = "not available from provider payload"
if revenue_growth is None:
unavailable["revenue_growth"] = "not available from provider payload"
if earnings_surprise is None:
unavailable["earnings_surprise"] = "not available from provider payload"
if market_cap is None:
unavailable["market_cap"] = "not available from provider payload"
return FundamentalData(
ticker=ticker,
pe_ratio=pe_ratio,
revenue_growth=revenue_growth,
earnings_surprise=earnings_surprise,
market_cap=market_cap,
fetched_at=datetime.now(timezone.utc),
next_earnings_date=next_earnings_date,
unavailable_fields=unavailable,
)
@staticmethod
def _next_earnings(resp: httpx.Response) -> date | None:
"""Earliest upcoming earnings date from Finnhub's calendar payload."""
try:
payload = resp.json() if resp.text else {}
except ValueError:
return None
entries = payload.get("earningsCalendar", []) if isinstance(payload, dict) else []
dates: list[date] = []
today = date.today()
for entry in entries if isinstance(entries, list) else []:
raw = entry.get("date") if isinstance(entry, dict) else None
if not raw:
continue
try:
parsed = date.fromisoformat(raw)
except ValueError:
continue
if parsed >= today:
dates.append(parsed)
return min(dates) if dates else None
class AlphaVantageFundamentalProvider:
"""Fundamentals provider backed by Alpha Vantage free endpoints."""
def __init__(self, api_key: str) -> None:
if not api_key:
raise ProviderError("Alpha Vantage API key is required")
self._api_key = api_key
self._base_url = "https://www.alphavantage.co/query"
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
unavailable: dict[str, str] = {}
api_symbol = _to_api_symbol(ticker)
async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client:
overview_resp = await client.get(
self._base_url,
params={"function": "OVERVIEW", "symbol": api_symbol, "apikey": self._api_key},
)
earnings_resp = await client.get(
self._base_url,
params={"function": "EARNINGS", "symbol": api_symbol, "apikey": self._api_key},
)
income_resp = await client.get(
self._base_url,
params={"function": "INCOME_STATEMENT", "symbol": api_symbol, "apikey": self._api_key},
)
for resp, endpoint in (
(overview_resp, "OVERVIEW"),
(earnings_resp, "EARNINGS"),
(income_resp, "INCOME_STATEMENT"),
):
if resp.status_code == 429:
raise RateLimitError(f"Alpha Vantage rate limit hit for {ticker} ({endpoint})")
if resp.status_code != 200:
raise ProviderError(f"Alpha Vantage error for {ticker} ({endpoint}): HTTP {resp.status_code}")
overview = overview_resp.json() if overview_resp.text else {}
earnings = earnings_resp.json() if earnings_resp.text else {}
income = income_resp.json() if income_resp.text else {}
if isinstance(overview, dict) and overview.get("Information"):
raise ProviderError(f"Alpha Vantage unavailable for {ticker}: {overview.get('Information')}")
if isinstance(overview, dict) and overview.get("Note"):
raise RateLimitError(f"Alpha Vantage rate limit for {ticker}: {overview.get('Note')}")
pe_ratio = _safe_float((overview or {}).get("PERatio"))
market_cap = _safe_float((overview or {}).get("MarketCapitalization"))
earnings_surprise = None
quarterly = earnings.get("quarterlyEarnings", []) if isinstance(earnings, dict) else []
if isinstance(quarterly, list) and quarterly:
first = quarterly[0] if isinstance(quarterly[0], dict) else {}
earnings_surprise = _safe_float(first.get("surprisePercentage"))
revenue_growth = None
annual = income.get("annualReports", []) if isinstance(income, dict) else []
if isinstance(annual, list) and len(annual) >= 2:
curr = _safe_float((annual[0] or {}).get("totalRevenue"))
prev = _safe_float((annual[1] or {}).get("totalRevenue"))
if curr is not None and prev not in (None, 0):
revenue_growth = ((curr - prev) / abs(prev)) * 100.0
if pe_ratio is None:
unavailable["pe_ratio"] = "not available from provider payload"
if revenue_growth is None:
unavailable["revenue_growth"] = "not available from provider payload"
if earnings_surprise is None:
unavailable["earnings_surprise"] = "not available from provider payload"
if market_cap is None:
unavailable["market_cap"] = "not available from provider payload"
return FundamentalData(
ticker=ticker,
pe_ratio=pe_ratio,
revenue_growth=revenue_growth,
earnings_surprise=earnings_surprise,
market_cap=market_cap,
fetched_at=datetime.now(timezone.utc),
unavailable_fields=unavailable,
)
_FUNDAMENTAL_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise", "market_cap")
class ChainedFundamentalProvider:
"""Merge fundamentals across providers, filling gaps from later sources.
A single provider rarely covers everything on free tiers — FMP's free plan,
for example, returns only market cap (the ratios/growth/earnings endpoints
402). Rather than stop at the first provider with *any* field, we take each
field from the first provider that supplies it, so FMP's market cap is
combined with Finnhub's P/E and earnings surprise.
"""
def __init__(self, providers: list[tuple[str, FundamentalProvider]]) -> None:
if not providers:
raise ProviderError("No fundamental providers configured")
self._providers = providers
async def fetch_fundamentals(self, ticker: str, allow_partial: bool = False) -> FundamentalData:
"""Merge fundamentals across providers.
``allow_partial`` controls behaviour when a fallback provider is *rate
limited* and we end up with missing fields. By default we raise
RateLimitError so the caller (the bulk collector) can back off and retry
the ticker once the window frees — otherwise a transient 429 on Finnhub
would be silently stored as market-cap-only. Pass ``allow_partial=True``
(manual single fetches, or the collector's final give-up attempt) to
accept whatever was gathered instead of raising.
"""
merged: dict[str, float | None] = {f: None for f in _FUNDAMENTAL_FIELDS}
field_source: dict[str, str] = {}
errors: list[str] = []
rate_limited = False
next_earnings_date = None
for provider_name, provider in self._providers:
if all(merged[f] is not None for f in _FUNDAMENTAL_FIELDS) and next_earnings_date:
break
try:
data = await provider.fetch_fundamentals(ticker)
except RateLimitError as exc:
rate_limited = True
errors.append(f"{provider_name}: RateLimitError: {exc}")
continue
except Exception as exc:
errors.append(f"{provider_name}: {type(exc).__name__}: {exc}")
continue
if next_earnings_date is None and data.next_earnings_date is not None:
next_earnings_date = data.next_earnings_date
for field in _FUNDAMENTAL_FIELDS:
if merged[field] is None:
value = getattr(data, field)
if value is not None:
merged[field] = value
field_source[field] = provider_name
missing = [f for f in _FUNDAMENTAL_FIELDS if merged[f] is None]
# A rate limit left data incomplete: signal it (unless partial is OK) so
# the collector backs off rather than persisting a degraded record.
if rate_limited and missing and not allow_partial:
attempts = "; ".join(errors[:6])
raise RateLimitError(
f"Fundamentals incomplete for {ticker} due to provider rate limits "
f"(missing {', '.join(missing)}). Attempts: {attempts}"
)
if all(merged[f] is None for f in _FUNDAMENTAL_FIELDS):
attempts = "; ".join(errors[:6]) if errors else "no usable metrics from any provider"
raise ProviderError(f"All fundamentals providers failed for {ticker}. Attempts: {attempts}")
unavailable: dict[str, str] = {
field: "not available from any configured provider"
for field in _FUNDAMENTAL_FIELDS
if merged[field] is None
}
# Record which provider supplied each field for transparency.
for field, src in field_source.items():
unavailable[f"source_{field}"] = src
return FundamentalData(
ticker=ticker,
pe_ratio=merged["pe_ratio"],
revenue_growth=merged["revenue_growth"],
earnings_surprise=merged["earnings_surprise"],
market_cap=merged["market_cap"],
fetched_at=datetime.now(timezone.utc),
next_earnings_date=next_earnings_date,
unavailable_fields=unavailable,
)
def build_fundamental_provider_chain() -> FundamentalProvider:
providers: list[tuple[str, FundamentalProvider]] = []
if settings.fmp_api_key:
providers.append(("fmp", FMPFundamentalProvider(settings.fmp_api_key)))
if settings.finnhub_api_key:
providers.append(("finnhub", FinnhubFundamentalProvider(settings.finnhub_api_key)))
if settings.alpha_vantage_api_key:
providers.append(("alpha_vantage", AlphaVantageFundamentalProvider(settings.alpha_vantage_api_key)))
if not providers:
raise ProviderError(
"No fundamentals provider configured. Set one of FMP_API_KEY, FINNHUB_API_KEY, ALPHA_VANTAGE_API_KEY"
)
logger.info("Fundamentals provider chain configured: %s", [name for name, _ in providers])
return ChainedFundamentalProvider(providers)
+2 -20
View File
@@ -44,20 +44,6 @@ class SentimentData:
recommendation: str | None = None # "buy" | "hold" | "avoid" — actionable LLM view
@dataclass(frozen=True, slots=True)
class FundamentalData:
"""Fundamental metrics returned by fundamental providers."""
ticker: str
pe_ratio: float | None
revenue_growth: float | None
earnings_surprise: float | None
market_cap: float | None
fetched_at: datetime
next_earnings_date: date | None = None
unavailable_fields: dict[str, str] = field(default_factory=dict)
# ---------------------------------------------------------------------------
# Provider Protocols
# ---------------------------------------------------------------------------
@@ -81,9 +67,5 @@ class SentimentProvider(Protocol):
...
class FundamentalProvider(Protocol):
"""Protocol for fundamental data providers."""
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
"""Fetch fundamental data for a ticker."""
...
# No fundamentals provider protocol: since A6 fundamentals come only from the
# batch SEC/Dolt imports, never from a request-time provider call.