fix bulk fundamentals: rate limits masked by partial FMP success
Root cause of "price plan needed in bulk but fine on manual reload": on free tiers FMP returns only market cap (others 402) and the chain merged that as a partial success — so when the Finnhub/Alpha Vantage fallbacks were rate-limited during a bulk run, the chain silently returned market-cap-only and the collector's backoff never engaged. Manual single fetches worked because the fallbacks weren't throttled at that moment. Fixes: - Chain distinguishes RateLimitError from other failures: if a fallback is rate-limited and fields are still missing, raise RateLimitError (unless allow_partial=True) so the collector backs off and retries. - Bulk job paces requests (fundamental_request_spacing_seconds, default 3s) to stay under Finnhub's ~60/min, and on retry-exhaustion stores partial data and continues instead of aborting the whole run. - Manual fetch passes allow_partial=True so a lone 429 doesn't fail the refresh. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -6,7 +6,7 @@ from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from app.exceptions import ProviderError
|
||||
from app.exceptions import ProviderError, RateLimitError
|
||||
from app.providers.fundamentals_chain import ChainedFundamentalProvider
|
||||
from app.providers.protocol import FundamentalData
|
||||
|
||||
@@ -19,6 +19,11 @@ class _FailProvider:
|
||||
raise ProviderError(f"{self._message} ({ticker})")
|
||||
|
||||
|
||||
class _RateLimitedProvider:
|
||||
async def fetch_fundamentals(self, ticker: str) -> FundamentalData:
|
||||
raise RateLimitError(f"rate limit hit for {ticker}")
|
||||
|
||||
|
||||
class _DataProvider:
|
||||
def __init__(self, data: FundamentalData) -> None:
|
||||
self._data = data
|
||||
@@ -98,3 +103,53 @@ async def test_chained_provider_raises_when_all_providers_fail():
|
||||
await provider.fetch_fundamentals("MSFT")
|
||||
|
||||
assert "All fundamentals providers failed" in str(exc.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limited_fallback_raises_when_incomplete():
|
||||
"""FMP gives market cap; the fallback is rate-limited → chain signals it so
|
||||
the collector can back off instead of storing a degraded record."""
|
||||
primary_data = FundamentalData(
|
||||
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
|
||||
market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
|
||||
)
|
||||
provider = ChainedFundamentalProvider([
|
||||
("fmp", _DataProvider(primary_data)),
|
||||
("finnhub", _RateLimitedProvider()),
|
||||
])
|
||||
|
||||
with pytest.raises(RateLimitError):
|
||||
await provider.fetch_fundamentals("AAPL")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limited_fallback_allows_partial():
|
||||
"""With allow_partial=True the chain returns the market cap it did get."""
|
||||
primary_data = FundamentalData(
|
||||
ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None,
|
||||
market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
|
||||
)
|
||||
provider = ChainedFundamentalProvider([
|
||||
("fmp", _DataProvider(primary_data)),
|
||||
("finnhub", _RateLimitedProvider()),
|
||||
])
|
||||
|
||||
result = await provider.fetch_fundamentals("AAPL", allow_partial=True)
|
||||
assert result.market_cap == 2_000_000.0
|
||||
assert result.pe_ratio is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rate_limited_but_complete_does_not_raise():
|
||||
"""If every field is filled, a rate limit on a later (unused) provider is moot."""
|
||||
full = FundamentalData(
|
||||
ticker="AAPL", pe_ratio=20.0, revenue_growth=10.0, earnings_surprise=2.0,
|
||||
market_cap=5.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={},
|
||||
)
|
||||
provider = ChainedFundamentalProvider([
|
||||
("fmp", _DataProvider(full)),
|
||||
("finnhub", _RateLimitedProvider()),
|
||||
])
|
||||
|
||||
result = await provider.fetch_fundamentals("AAPL")
|
||||
assert result.pe_ratio == 20.0
|
||||
|
||||
Reference in New Issue
Block a user