fix: convert Finnhub market cap from millions to absolute USD

Finnhub profile2 reports marketCapitalization in millions; storing it
as dollars made mega-caps like SPCX show as micro (e.g. 1.8M). Normalize
on ingest, add unit tests, and include a one-shot SQL backfill script.

app/providers/fundamentals_chain.py

@@ -101,7 +101,10 @@ class FinnhubFundamentalProvider:
         earnings_payload = earnings_resp.json() if earnings_resp.text else []
 
         metrics = metric_payload.get("metric", {}) if isinstance(metric_payload, dict) else {}
-        market_cap = _safe_float((profile_payload or {}).get("marketCapitalization"))
+        # Finnhub profile2 marketCapitalization is in millions of USD.
+        # Normalize to absolute dollars so cap bands / formatters match FMP & Alpha Vantage.
+        market_cap_millions = _safe_float((profile_payload or {}).get("marketCapitalization"))
+        market_cap = market_cap_millions * 1_000_000.0 if market_cap_millions is not None else None
         pe_ratio = _safe_float(metrics.get("peTTM") or metrics.get("peNormalizedAnnual"))
         revenue_growth = _safe_float(metrics.get("revenueGrowthTTMYoy") or metrics.get("revenueGrowth5Y"))
 

scripts/fix_finnhub_market_cap.sql

@@ -0,0 +1,113 @@
+-- Fix Finnhub market caps stored in millions instead of absolute USD.
+--
+-- Context:
+--   Finnhub profile2.marketCapitalization is in millions of USD.
+--   Pre-fix rows stored that value as-is (e.g. SPCX ~1.8e6 → "1.8M micro").
+--   Correct absolute USD is value * 1_000_000 (e.g. ~1.8e12 → "1.8T mega").
+--
+-- Identification:
+--   Chained fundamentals write source provenance into unavailable_fields_json:
+--     "source_market_cap": "finnhub"
+--   FMP / Alpha Vantage rows are already absolute USD — leave them alone.
+--
+-- Safety:
+--   After a successful update we stamp market_cap_unit = "usd" so this script
+--   is idempotent (safe to re-run).
+--
+-- Database: PostgreSQL (production).
+-- Run against your app DB, e.g.:
+--   psql "$DATABASE_URL" -f scripts/fix_finnhub_market_cap.sql
+-- or step through the sections in a SQL client.
+
+-- ---------------------------------------------------------------------------
+-- 1) Preview: who would be fixed
+-- ---------------------------------------------------------------------------
+SELECT
+    t.symbol,
+    f.market_cap                          AS stored_millions_as_dollars,
+    f.market_cap * 1000000.0              AS corrected_usd,
+    CASE
+        WHEN f.market_cap * 1000000.0 >= 200e9 THEN 'mega'
+        WHEN f.market_cap * 1000000.0 >= 10e9  THEN 'large'
+        WHEN f.market_cap * 1000000.0 >= 2e9   THEN 'mid'
+        WHEN f.market_cap * 1000000.0 >= 300e6 THEN 'small'
+        ELSE 'micro'
+    END                                   AS corrected_band,
+    f.fetched_at,
+    f.unavailable_fields_json
+FROM fundamental_data f
+JOIN tickers t ON t.id = f.ticker_id
+WHERE f.market_cap IS NOT NULL
+  AND f.market_cap > 0
+  AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
+  AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd'
+ORDER BY f.market_cap DESC;
+
+-- Count
+SELECT COUNT(*) AS rows_to_fix
+FROM fundamental_data f
+WHERE f.market_cap IS NOT NULL
+  AND f.market_cap > 0
+  AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
+  AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd';
+
+-- ---------------------------------------------------------------------------
+-- 2) Apply fix (run inside a transaction; COMMIT only if preview looks right)
+-- ---------------------------------------------------------------------------
+BEGIN;
+
+UPDATE fundamental_data f
+SET
+    market_cap = f.market_cap * 1000000.0,
+    unavailable_fields_json = (
+        f.unavailable_fields_json::jsonb || '{"market_cap_unit": "usd"}'::jsonb
+    )::text
+WHERE f.market_cap IS NOT NULL
+  AND f.market_cap > 0
+  AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
+  AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd';
+
+-- Sanity check after update (should include SPCX ~1e12 scale if present)
+SELECT
+    t.symbol,
+    f.market_cap,
+    f.unavailable_fields_json
+FROM fundamental_data f
+JOIN tickers t ON t.id = f.ticker_id
+WHERE f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
+  AND f.unavailable_fields_json::jsonb->>'market_cap_unit' = 'usd'
+ORDER BY f.market_cap DESC NULLS LAST
+LIMIT 30;
+
+-- Remaining unfixed Finnhub rows (should be 0)
+SELECT COUNT(*) AS remaining_unfixed
+FROM fundamental_data f
+WHERE f.market_cap IS NOT NULL
+  AND f.market_cap > 0
+  AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
+  AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd';
+
+-- COMMIT;   -- uncomment when satisfied
+-- ROLLBACK; -- or roll back if something looks off
+
+-- ---------------------------------------------------------------------------
+-- Optional: rows with market_cap but NO source_market_cap provenance.
+-- These cannot be safely auto-fixed (could be FMP absolute USD already).
+-- Inspect manually if anything looks like Finnhub-scale millions.
+-- ---------------------------------------------------------------------------
+-- SELECT
+--     t.symbol,
+--     f.market_cap,
+--     f.unavailable_fields_json,
+--     f.fetched_at
+-- FROM fundamental_data f
+-- JOIN tickers t ON t.id = f.ticker_id
+-- WHERE f.market_cap IS NOT NULL
+--   AND f.market_cap > 0
+--   AND (
+--         f.unavailable_fields_json IS NULL
+--      OR f.unavailable_fields_json = '{}'
+--      OR f.unavailable_fields_json::jsonb->>'source_market_cap' IS NULL
+--   )
+-- ORDER BY f.market_cap ASC
+-- LIMIT 50;

tests/unit/test_finnhub_provider.py

@@ -0,0 +1,92 @@
+"""Unit tests for FinnhubFundamentalProvider unit conversions."""
+
+from __future__ import annotations
+
+from unittest.mock import AsyncMock, patch
+
+import httpx
+import pytest
+
+from app.providers.fundamentals_chain import FinnhubFundamentalProvider
+
+
+def _mock_response(status_code: int, json_data: object = None) -> httpx.Response:
+    return httpx.Response(
+        status_code=status_code,
+        json=json_data if json_data is not None else {},
+        request=httpx.Request("GET", "https://example.com"),
+    )
+
+
+@pytest.fixture
+def provider() -> FinnhubFundamentalProvider:
+    return FinnhubFundamentalProvider(api_key="test-key")
+
+
+@pytest.mark.asyncio
+async def test_finnhub_market_cap_converted_from_millions_to_dollars(provider):
+    """Finnhub marketCapitalization is in millions — store absolute USD.
+
+    SPCX-scale example: ~$1.8T → Finnhub reports 1_800_000 (millions).
+    Without conversion the UI showed 1.8M / micro cap.
+    """
+    profile = {"marketCapitalization": 1_800_000}  # millions → $1.8T
+    metrics = {"metric": {"peTTM": 40.0, "revenueGrowthTTMYoy": 25.0}}
+    earnings = [{"surprisePercent": 2.5}]
+    calendar = {"earningsCalendar": []}
+
+    async def mock_get(url, params=None):
+        if "profile2" in url:
+            return _mock_response(200, profile)
+        if "stock/metric" in url:
+            return _mock_response(200, metrics)
+        if "stock/earnings" in url:
+            return _mock_response(200, earnings)
+        if "calendar/earnings" in url:
+            return _mock_response(200, calendar)
+        return _mock_response(200, {})
+
+    with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient:
+        instance = AsyncMock()
+        instance.get.side_effect = mock_get
+        instance.__aenter__ = AsyncMock(return_value=instance)
+        instance.__aexit__ = AsyncMock(return_value=False)
+        MockClient.return_value = instance
+
+        result = await provider.fetch_fundamentals("SPCX")
+
+    assert result.market_cap == 1_800_000 * 1_000_000  # $1.8T
+    assert result.pe_ratio == 40.0
+    assert result.revenue_growth == 25.0
+    assert result.earnings_surprise == 2.5
+
+
+@pytest.mark.asyncio
+async def test_finnhub_market_cap_none_when_missing(provider):
+    profile: dict = {}
+    metrics = {"metric": {}}
+    earnings: list = []
+    calendar = {"earningsCalendar": []}
+
+    async def mock_get(url, params=None):
+        if "profile2" in url:
+            return _mock_response(200, profile)
+        if "stock/metric" in url:
+            return _mock_response(200, metrics)
+        if "stock/earnings" in url:
+            return _mock_response(200, earnings)
+        if "calendar/earnings" in url:
+            return _mock_response(200, calendar)
+        return _mock_response(200, {})
+
+    with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient:
+        instance = AsyncMock()
+        instance.get.side_effect = mock_get
+        instance.__aenter__ = AsyncMock(return_value=instance)
+        instance.__aexit__ = AsyncMock(return_value=False)
+        MockClient.return_value = instance
+
+        result = await provider.fetch_fundamentals("XYZ")
+
+    assert result.market_cap is None
+    assert "market_cap" in result.unavailable_fields