dennisthiessen
5a531fd603
Finnhub profile2 reports marketCapitalization in millions; storing it as dollars made mega-caps like SPCX show as micro (e.g. 1.8M). Normalize on ingest, add unit tests, and include a one-shot SQL backfill script.
114 lines
4.3 KiB
PL/PgSQL
114 lines
4.3 KiB
PL/PgSQL
-- Fix Finnhub market caps stored in millions instead of absolute USD.
|
|
--
|
|
-- Context:
|
|
-- Finnhub profile2.marketCapitalization is in millions of USD.
|
|
-- Pre-fix rows stored that value as-is (e.g. SPCX ~1.8e6 → "1.8M micro").
|
|
-- Correct absolute USD is value * 1_000_000 (e.g. ~1.8e12 → "1.8T mega").
|
|
--
|
|
-- Identification:
|
|
-- Chained fundamentals write source provenance into unavailable_fields_json:
|
|
-- "source_market_cap": "finnhub"
|
|
-- FMP / Alpha Vantage rows are already absolute USD — leave them alone.
|
|
--
|
|
-- Safety:
|
|
-- After a successful update we stamp market_cap_unit = "usd" so this script
|
|
-- is idempotent (safe to re-run).
|
|
--
|
|
-- Database: PostgreSQL (production).
|
|
-- Run against your app DB, e.g.:
|
|
-- psql "$DATABASE_URL" -f scripts/fix_finnhub_market_cap.sql
|
|
-- or step through the sections in a SQL client.
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 1) Preview: who would be fixed
|
|
-- ---------------------------------------------------------------------------
|
|
SELECT
|
|
t.symbol,
|
|
f.market_cap AS stored_millions_as_dollars,
|
|
f.market_cap * 1000000.0 AS corrected_usd,
|
|
CASE
|
|
WHEN f.market_cap * 1000000.0 >= 200e9 THEN 'mega'
|
|
WHEN f.market_cap * 1000000.0 >= 10e9 THEN 'large'
|
|
WHEN f.market_cap * 1000000.0 >= 2e9 THEN 'mid'
|
|
WHEN f.market_cap * 1000000.0 >= 300e6 THEN 'small'
|
|
ELSE 'micro'
|
|
END AS corrected_band,
|
|
f.fetched_at,
|
|
f.unavailable_fields_json
|
|
FROM fundamental_data f
|
|
JOIN tickers t ON t.id = f.ticker_id
|
|
WHERE f.market_cap IS NOT NULL
|
|
AND f.market_cap > 0
|
|
AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
|
|
AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd'
|
|
ORDER BY f.market_cap DESC;
|
|
|
|
-- Count
|
|
SELECT COUNT(*) AS rows_to_fix
|
|
FROM fundamental_data f
|
|
WHERE f.market_cap IS NOT NULL
|
|
AND f.market_cap > 0
|
|
AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
|
|
AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd';
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- 2) Apply fix (run inside a transaction; COMMIT only if preview looks right)
|
|
-- ---------------------------------------------------------------------------
|
|
BEGIN;
|
|
|
|
UPDATE fundamental_data f
|
|
SET
|
|
market_cap = f.market_cap * 1000000.0,
|
|
unavailable_fields_json = (
|
|
f.unavailable_fields_json::jsonb || '{"market_cap_unit": "usd"}'::jsonb
|
|
)::text
|
|
WHERE f.market_cap IS NOT NULL
|
|
AND f.market_cap > 0
|
|
AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
|
|
AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd';
|
|
|
|
-- Sanity check after update (should include SPCX ~1e12 scale if present)
|
|
SELECT
|
|
t.symbol,
|
|
f.market_cap,
|
|
f.unavailable_fields_json
|
|
FROM fundamental_data f
|
|
JOIN tickers t ON t.id = f.ticker_id
|
|
WHERE f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
|
|
AND f.unavailable_fields_json::jsonb->>'market_cap_unit' = 'usd'
|
|
ORDER BY f.market_cap DESC NULLS LAST
|
|
LIMIT 30;
|
|
|
|
-- Remaining unfixed Finnhub rows (should be 0)
|
|
SELECT COUNT(*) AS remaining_unfixed
|
|
FROM fundamental_data f
|
|
WHERE f.market_cap IS NOT NULL
|
|
AND f.market_cap > 0
|
|
AND f.unavailable_fields_json::jsonb->>'source_market_cap' = 'finnhub'
|
|
AND COALESCE(f.unavailable_fields_json::jsonb->>'market_cap_unit', '') <> 'usd';
|
|
|
|
-- COMMIT; -- uncomment when satisfied
|
|
-- ROLLBACK; -- or roll back if something looks off
|
|
|
|
-- ---------------------------------------------------------------------------
|
|
-- Optional: rows with market_cap but NO source_market_cap provenance.
|
|
-- These cannot be safely auto-fixed (could be FMP absolute USD already).
|
|
-- Inspect manually if anything looks like Finnhub-scale millions.
|
|
-- ---------------------------------------------------------------------------
|
|
-- SELECT
|
|
-- t.symbol,
|
|
-- f.market_cap,
|
|
-- f.unavailable_fields_json,
|
|
-- f.fetched_at
|
|
-- FROM fundamental_data f
|
|
-- JOIN tickers t ON t.id = f.ticker_id
|
|
-- WHERE f.market_cap IS NOT NULL
|
|
-- AND f.market_cap > 0
|
|
-- AND (
|
|
-- f.unavailable_fields_json IS NULL
|
|
-- OR f.unavailable_fields_json = '{}'
|
|
-- OR f.unavailable_fields_json::jsonb->>'source_market_cap' IS NULL
|
|
-- )
|
|
-- ORDER BY f.market_cap ASC
|
|
-- LIMIT 50;
|