diff --git a/app/providers/fundamentals_chain.py b/app/providers/fundamentals_chain.py index d2f4896..cba5e85 100644 --- a/app/providers/fundamentals_chain.py +++ b/app/providers/fundamentals_chain.py @@ -101,7 +101,10 @@ class FinnhubFundamentalProvider: earnings_payload = earnings_resp.json() if earnings_resp.text else [] metrics = metric_payload.get("metric", {}) if isinstance(metric_payload, dict) else {} - market_cap = _safe_float((profile_payload or {}).get("marketCapitalization")) + # Finnhub profile2 marketCapitalization is in millions of USD. + # Normalize to absolute dollars so cap bands / formatters match FMP & Alpha Vantage. + market_cap_millions = _safe_float((profile_payload or {}).get("marketCapitalization")) + market_cap = market_cap_millions * 1_000_000.0 if market_cap_millions is not None else None pe_ratio = _safe_float(metrics.get("peTTM") or metrics.get("peNormalizedAnnual")) revenue_growth = _safe_float(metrics.get("revenueGrowthTTMYoy") or metrics.get("revenueGrowth5Y")) diff --git a/scripts/fix_finnhub_market_cap.sql b/scripts/fix_finnhub_market_cap.sql new file mode 100644 index 0000000..af3d867 --- /dev/null +++ b/scripts/fix_finnhub_market_cap.sql @@ -0,0 +1,113 @@ +-- Fix Finnhub market caps stored in millions instead of absolute USD. +-- +-- Context: +-- Finnhub profile2.marketCapitalization is in millions of USD. +-- Pre-fix rows stored that value as-is (e.g. SPCX ~1.8e6 → "1.8M micro"). +-- Correct absolute USD is value * 1_000_000 (e.g. ~1.8e12 → "1.8T mega"). +-- +-- Identification: +-- Chained fundamentals write source provenance into unavailable_fields_json: +-- "source_market_cap": "finnhub" +-- FMP / Alpha Vantage rows are already absolute USD — leave them alone. +-- +-- Safety: +-- After a successful update we stamp market_cap_unit = "usd" so this script +-- is idempotent (safe to re-run). +-- +-- Database: PostgreSQL (production). +-- Run against your app DB, e.g.: +-- psql "$DATABASE_URL" -f scripts/fix_finnhub_market_cap.sql +-- or step through the sections in a SQL client. + +-- --------------------------------------------------------------------------- +-- 1) Preview: who would be fixed +-- --------------------------------------------------------------------------- +SELECT + t.symbol, + f.market_cap AS stored_millions_as_dollars, + f.market_cap * 1000000.0 AS corrected_usd, + CASE + WHEN f.market_cap * 1000000.0 >= 200e9 THEN 'mega' + WHEN f.market_cap * 1000000.0 >= 10e9 THEN 'large' + WHEN f.market_cap * 1000000.0 >= 2e9 THEN 'mid' + WHEN f.market_cap * 1000000.0 >= 300e6 THEN 'small' + ELSE 'micro' + END AS corrected_band, + f.fetched_at, + f.unavailable_fields_json +FROM fundamental_data f +JOIN tickers t ON t.id = f.ticker_id +WHERE f.market_cap IS NOT NULL + AND f.market_cap > 0 + AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub' + AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd' +ORDER BY f.market_cap DESC; + +-- Count +SELECT COUNT(*) AS rows_to_fix +FROM fundamental_data f +WHERE f.market_cap IS NOT NULL + AND f.market_cap > 0 + AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub' + AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd'; + +-- --------------------------------------------------------------------------- +-- 2) Apply fix (run inside a transaction; COMMIT only if preview looks right) +-- --------------------------------------------------------------------------- +BEGIN; + +UPDATE fundamental_data f +SET + market_cap = f.market_cap * 1000000.0, + unavailable_fields_json = ( + f.unavailable_fields_json::jsonb || '{"market_cap_unit": "usd"}'::jsonb + )::text +WHERE f.market_cap IS NOT NULL + AND f.market_cap > 0 + AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub' + AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd'; + +-- Sanity check after update (should include SPCX ~1e12 scale if present) +SELECT + t.symbol, + f.market_cap, + f.unavailable_fields_json +FROM fundamental_data f +JOIN tickers t ON t.id = f.ticker_id +WHERE f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub' + AND f.unavailable_fields_json::jsonb->>'market_cap_unit' = 'usd' +ORDER BY f.market_cap DESC NULLS LAST +LIMIT 30; + +-- Remaining unfixed Finnhub rows (should be 0) +SELECT COUNT(*) AS remaining_unfixed +FROM fundamental_data f +WHERE f.market_cap IS NOT NULL + AND f.market_cap > 0 + AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub' + AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd'; + +-- COMMIT; -- uncomment when satisfied +-- ROLLBACK; -- or roll back if something looks off + +-- --------------------------------------------------------------------------- +-- Optional: rows with market_cap but NO source_market_cap provenance. +-- These cannot be safely auto-fixed (could be FMP absolute USD already). +-- Inspect manually if anything looks like Finnhub-scale millions. +-- --------------------------------------------------------------------------- +-- SELECT +-- t.symbol, +-- f.market_cap, +-- f.unavailable_fields_json, +-- f.fetched_at +-- FROM fundamental_data f +-- JOIN tickers t ON t.id = f.ticker_id +-- WHERE f.market_cap IS NOT NULL +-- AND f.market_cap > 0 +-- AND ( +-- f.unavailable_fields_json IS NULL +-- OR f.unavailable_fields_json = '{}' +-- OR f.unavailable_fields_json::jsonb->>'source_market_cap' IS NULL +-- ) +-- ORDER BY f.market_cap ASC +-- LIMIT 50; diff --git a/tests/unit/test_finnhub_provider.py b/tests/unit/test_finnhub_provider.py new file mode 100644 index 0000000..ab5ec21 --- /dev/null +++ b/tests/unit/test_finnhub_provider.py @@ -0,0 +1,92 @@ +"""Unit tests for FinnhubFundamentalProvider unit conversions.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, patch + +import httpx +import pytest + +from app.providers.fundamentals_chain import FinnhubFundamentalProvider + + +def _mock_response(status_code: int, json_data: object = None) -> httpx.Response: + return httpx.Response( + status_code=status_code, + json=json_data if json_data is not None else {}, + request=httpx.Request("GET", "https://example.com"), + ) + + +@pytest.fixture +def provider() -> FinnhubFundamentalProvider: + return FinnhubFundamentalProvider(api_key="test-key") + + +@pytest.mark.asyncio +async def test_finnhub_market_cap_converted_from_millions_to_dollars(provider): + """Finnhub marketCapitalization is in millions — store absolute USD. + + SPCX-scale example: ~$1.8T → Finnhub reports 1_800_000 (millions). + Without conversion the UI showed 1.8M / micro cap. + """ + profile = {"marketCapitalization": 1_800_000} # millions → $1.8T + metrics = {"metric": {"peTTM": 40.0, "revenueGrowthTTMYoy": 25.0}} + earnings = [{"surprisePercent": 2.5}] + calendar = {"earningsCalendar": []} + + async def mock_get(url, params=None): + if "profile2" in url: + return _mock_response(200, profile) + if "stock/metric" in url: + return _mock_response(200, metrics) + if "stock/earnings" in url: + return _mock_response(200, earnings) + if "calendar/earnings" in url: + return _mock_response(200, calendar) + return _mock_response(200, {}) + + with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient: + instance = AsyncMock() + instance.get.side_effect = mock_get + instance.__aenter__ = AsyncMock(return_value=instance) + instance.__aexit__ = AsyncMock(return_value=False) + MockClient.return_value = instance + + result = await provider.fetch_fundamentals("SPCX") + + assert result.market_cap == 1_800_000 * 1_000_000 # $1.8T + assert result.pe_ratio == 40.0 + assert result.revenue_growth == 25.0 + assert result.earnings_surprise == 2.5 + + +@pytest.mark.asyncio +async def test_finnhub_market_cap_none_when_missing(provider): + profile: dict = {} + metrics = {"metric": {}} + earnings: list = [] + calendar = {"earningsCalendar": []} + + async def mock_get(url, params=None): + if "profile2" in url: + return _mock_response(200, profile) + if "stock/metric" in url: + return _mock_response(200, metrics) + if "stock/earnings" in url: + return _mock_response(200, earnings) + if "calendar/earnings" in url: + return _mock_response(200, calendar) + return _mock_response(200, {}) + + with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient: + instance = AsyncMock() + instance.get.side_effect = mock_get + instance.__aenter__ = AsyncMock(return_value=instance) + instance.__aexit__ = AsyncMock(return_value=False) + MockClient.return_value = instance + + result = await provider.fetch_fundamentals("XYZ") + + assert result.market_cap is None + assert "market_cap" in result.unavailable_fields