diff --git a/.env.example b/.env.example index 31cb036..ed303b4 100644 --- a/.env.example +++ b/.env.example @@ -18,16 +18,9 @@ OPENAI_API_KEY= OPENAI_MODEL=gpt-4o-mini OPENAI_SENTIMENT_BATCH_SIZE=5 -# Fundamentals Provider — Financial Modeling Prep -FMP_API_KEY= - -# Fundamentals Provider — Finnhub (optional fallback) -FINNHUB_API_KEY= - -# Fundamentals Provider — Alpha Vantage (optional fallback) -ALPHA_VANTAGE_API_KEY= - -# Dolt bulk data — local clone of post-no-preference/earnings (workstream A). +# Dolt bulk data — local clone of post-no-preference/earnings. Together with the +# SEC EDGAR block below this is the ONLY fundamentals source; there is no +# provider-API fallback. # DOLT_BINARY: path to the dolt CLI (set the full path in dev if it's not on PATH, # e.g. Windows: C:\Program Files\Dolt\bin\dolt.exe). DOLT_DATA_DIR holds the # clones; in PRODUCTION it MUST be outside the deploy tree (deploy is @@ -52,10 +45,6 @@ SEC_REQUEST_SPACING_SECONDS=0.2 SEC_MAX_RETRIES=4 SEC_REQUEST_TIMEOUT_SECONDS=30.0 -# A5 read-only parity report archive. In production keep this outside the -# rsync deployment tree, e.g. /var/lib/signal-platform/reports/fundamentals-parity. -FUNDAMENTALS_PARITY_REPORT_DIR=reports/fundamentals-parity - # Regime Monitor — FRED (VIX + HY credit spreads). Free key: https://fred.stlouisfed.org/docs/api/api_key.html # Optional: without it the volatility (V1) and credit (C1) pillars show as n/a. FRED_API_KEY= @@ -63,10 +52,7 @@ FRED_API_KEY= # Scheduled Jobs DATA_COLLECTOR_FREQUENCY=daily SENTIMENT_POLL_INTERVAL_MINUTES=30 -FUNDAMENTAL_FETCH_FREQUENCY=daily RR_SCAN_FREQUENCY=daily -FUNDAMENTAL_RATE_LIMIT_RETRIES=3 -FUNDAMENTAL_RATE_LIMIT_BACKOFF_SECONDS=15 # Scoring Defaults DEFAULT_WATCHLIST_AUTO_SIZE=10 diff --git a/README.md b/README.md index fe90cf2..e11da7b 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ Hourly mid-session (Mon–Fri ~10:00–15:00 ET): only **OHLCV → Outcome Eval* ### Other jobs -Fundamentals (weekly, early Monday ET) · Backtest (weekly) · Ticker-universe sync (daily). Alerts auto-fire only via the near-close pipeline (still manually triggerable). Deep history backfill and event study are manual-only (Admin → Jobs). +Dolt earnings import (daily 02:30 ET) · SEC fundamentals import (daily 04:00 ET, also refreshes the fundamentals cache scoring reads) · Backtest (weekly) · Ticker-universe sync (daily). Alerts auto-fire only via the near-close pipeline (still manually triggerable). Deep history backfill and event study are manual-only (Admin → Jobs). ### From score to "top pick" @@ -301,13 +301,13 @@ Corollaries: never let an unvalidated score gate setups; the outcome evaluator m | Charts | Canvas 2D candlestick chart with S/R overlays | | Routing | React Router v6 (SPA) | | HTTP | Axios with JWT interceptor | -| Data providers | Alpaca (OHLCV); OpenAI / Gemini / DeepSeek / xAI (sentiment, pluggable); Fundamentals chain: FMP → Finnhub → Alpha Vantage; FRED (regime); Telegram (alerts) | +| Data providers | Alpaca (OHLCV); OpenAI / Gemini / DeepSeek / xAI (sentiment, pluggable); SEC EDGAR Company Facts + DoltHub earnings (fundamentals, bulk import); FRED (regime); Telegram (alerts) | ## Features ### Backend - Ticker registry with full cascade delete -- Universe bootstrap for `sp500`, `nasdaq100`, `nasdaq_all` via admin endpoint +- Universe bootstrap for `sp500`, `nasdaq100`, `nasdaq_all` via admin endpoint — free public sources (Wikipedia / NASDAQ Trader), then the cached snapshot, then a built-in seed list. The seeds are representative, not complete, so a *fresh* install bootstrapped while the public source is unreachable gets a partial universe; a warm instance falls through to its cache. - OHLCV price storage with upsert and validation - Technical indicators: ADX, EMA, RSI, ATR, Volume Profile, Pivot Points, EMA Cross - Structural Support/Resistance detection with rejection/recency strength, ATR-adaptive merging and a hard cap; persisted for charts and alerts @@ -583,18 +583,12 @@ Configure in `.env` (copy from `.env.example`): | `OPENAI_API_KEY` | For sentiment (OpenAI path) | — | OpenAI API key | | `OPENAI_MODEL` | No | `gpt-4o-mini` | OpenAI model name | | `OPENAI_SENTIMENT_BATCH_SIZE` | No | `5` | Micro-batch size for sentiment collector | -| `FMP_API_KEY` | Optional (fundamentals) | — | Financial Modeling Prep API key (first provider in chain) | -| `FINNHUB_API_KEY` | Optional (fundamentals) | — | Finnhub API key (fallback provider) | -| `ALPHA_VANTAGE_API_KEY` | Optional (fundamentals) | — | Alpha Vantage API key (fallback provider) | | `FRED_API_KEY` | Optional (regime) | — | FRED key for the regime monitor (VIX, credit spreads) | | `TELEGRAM_BOT_TOKEN` | Optional (alerts) | — | Telegram bot token for alerts (can also be set in Admin) | | `TELEGRAM_CHAT_ID` | Optional (alerts) | — | Telegram chat id for alerts | | `DATA_COLLECTOR_FREQUENCY` | No | `daily` | OHLCV collection schedule (legacy — see note below) | | `SENTIMENT_POLL_INTERVAL_MINUTES` | No | `30` | Sentiment polling interval | -| `FUNDAMENTAL_FETCH_FREQUENCY` | No | `weekly` | Fundamentals fetch cadence | | `RR_SCAN_FREQUENCY` | No | `daily` | R:R scanner schedule | -| `FUNDAMENTAL_RATE_LIMIT_RETRIES` | No | `3` | Retries per ticker on fundamentals rate-limit | -| `FUNDAMENTAL_RATE_LIMIT_BACKOFF_SECONDS` | No | `15` | Base backoff seconds for fundamentals retry (exponential) | | `DEFAULT_WATCHLIST_AUTO_SIZE` | No | `10` | Auto-watchlist size | | `DEFAULT_RR_THRESHOLD` | No | `1.5` | Minimum R:R ratio for setups | | `DB_POOL_SIZE` | No | `5` | Database connection pool size | diff --git a/alembic/versions/029_retire_legacy_fundamentals_settings.py b/alembic/versions/029_retire_legacy_fundamentals_settings.py new file mode 100644 index 0000000..04bb198 --- /dev/null +++ b/alembic/versions/029_retire_legacy_fundamentals_settings.py @@ -0,0 +1,96 @@ +"""Retire the legacy fundamentals settings (A6) + +Revision ID: 029 +Revises: 028 +Create Date: 2026-08-07 00:00:00.000000 + +A6 removed the FMP/Finnhub/Alpha Vantage providers, the weekly +``fundamental_collector`` job and the A5 parity report. Five SystemSetting rows +are left over. They are NOT all deleted, because the deploy runs migrations +before restarting the service: for a short window — and for the whole of any +rollback — pre-A6 code is still live, and it reads absent rows permissively +(cutover absent -> disabled; ``job__enabled`` absent -> enabled). Deleting +both would hand a rolled-back process a re-armed legacy collector writing over +the SEC/Dolt cache. + +So the two rows that carry behavior become tombstones pinned to the safe value, +and only the inert ones are deleted. The tombstones are dropped in a later +release once the rollback window has closed; ``SettingsForm`` hides them +meanwhile. +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + + +revision: str = "029" +down_revision: Union[str, None] = "028" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# Behavior-bearing under pre-A6 code -> pin to the safe value, keep the row. +_TOMBSTONES: dict[str, str] = { + "fundamental_data_sec_dolt_cutover_enabled": "true", + "job_fundamental_collector_enabled": "false", +} + +# Inert either way: an absent cron falls back to a default for a job that no +# longer registers, and the parity report never wrote anything. +_OBSOLETE: tuple[str, ...] = ( + "schedule_fundamentals_cron", + "schedule_fundamentals_parity_cron", + "job_fundamentals_parity_report_enabled", +) + +_settings = sa.table( + "system_settings", + sa.column("id", sa.Integer), + sa.column("key", sa.String), + sa.column("value", sa.Text), + sa.column("updated_at", sa.DateTime(timezone=True)), +) + + +def upgrade() -> None: + conn = op.get_bind() + now = sa.func.now() + + for key, pinned in _TOMBSTONES.items(): + row = conn.execute( + sa.select(_settings.c.value).where(_settings.c.key == key) + ).fetchone() + old_value = row[0] if row is not None else None + print(f"a6_tombstone {key}: {old_value!r} -> {pinned!r}", flush=True) + if row is None: + conn.execute( + sa.insert(_settings).values(key=key, value=pinned, updated_at=now) + ) + elif old_value != pinned: + conn.execute( + sa.update(_settings) + .where(_settings.c.key == key) + .values(value=pinned, updated_at=now) + ) + + # Print the value before deleting — a bare DELETE cannot be undone from the + # migration output. + for key in _OBSOLETE: + row = conn.execute( + sa.select(_settings.c.value).where(_settings.c.key == key) + ).fetchone() + if row is None: + print(f"a6_delete {key}: absent", flush=True) + continue + print(f"a6_delete {key}: {row[0]!r}", flush=True) + conn.execute(sa.delete(_settings).where(_settings.c.key == key)) + + +def downgrade() -> None: + """No-op. + + The deleted rows configured jobs this revision's code no longer registers, + and the tombstones already hold the values pre-A6 code needs. Recreating + them would restore nothing useful; the printed values above cover recovery. + """ diff --git a/app/config.py b/app/config.py index e43ef63..d1fb81e 100644 --- a/app/config.py +++ b/app/config.py @@ -28,15 +28,6 @@ class Settings(BaseSettings): deepseek_api_key: str = "" xai_api_key: str = "" - # Fundamentals Provider — Financial Modeling Prep - fmp_api_key: str = "" - - # Fundamentals Provider — Finnhub (optional fallback) - finnhub_api_key: str = "" - - # Fundamentals Provider — Alpha Vantage (optional fallback) - alpha_vantage_api_key: str = "" - # Dolt bulk-data — local clone of post-no-preference/earnings (workstream A). # dolt_binary: full path when not on PATH (dev/Windows install). dolt_data_dir # holds the clones; in production it MUST be outside the deploy tree (deploy is @@ -61,10 +52,6 @@ class Settings(BaseSettings): sec_max_retries: int = 4 sec_request_timeout_seconds: float = 30.0 - # A5 read-only comparison artifacts. Production must keep this outside the - # rsync deployment tree so the 5-7 day review window survives deploys. - fundamentals_parity_report_dir: str = "reports/fundamentals-parity" - # Regime Monitor — FRED (VIX level + HY credit spreads). Optional: without it # the volatility (P5) and credit-spread (F2) signals are reported as n/a. fred_api_key: str = "" @@ -86,15 +73,8 @@ class Settings(BaseSettings): # the score window is 7 days). sentiment_fresh_hours: int = 120 sentiment_top_composite: int = 30 - fundamental_fetch_frequency: str = "weekly" # quarterly-ish data; weekly conserves API quota rr_scan_frequency: str = "daily" # legacy label; qualifying scan is cron near-close # alerts_frequency removed: alerts fire only via morning + near-close pipelines - fundamental_rate_limit_retries: int = 3 - fundamental_rate_limit_backoff_seconds: int = 15 - # Pause between tickers in the bulk fundamentals job. Free tiers throttle - # hard (Finnhub ~60 calls/min, ~3 calls/ticker → ~3s/ticker); without - # spacing the job bursts straight into 429s. 0 disables. - fundamental_request_spacing_seconds: float = 3.0 # Scoring Defaults default_watchlist_auto_size: int = 10 diff --git a/app/providers/fmp.py b/app/providers/fmp.py deleted file mode 100644 index 15ec7c1..0000000 --- a/app/providers/fmp.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Financial Modeling Prep (FMP) fundamentals provider using httpx. - -Uses the stable API endpoints (https://financialmodelingprep.com/stable/) -which replaced the legacy /api/v3/ endpoints deprecated in Aug 2025. -""" - -from __future__ import annotations - -import logging -import os -from datetime import datetime, timezone -from pathlib import Path - -import httpx - -from app.exceptions import ProviderError, RateLimitError -from app.providers.protocol import FundamentalData - -logger = logging.getLogger(__name__) - -_FMP_STABLE_URL = "https://financialmodelingprep.com/stable" - -# Resolve CA bundle for explicit httpx verify -_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "") -if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists(): - _CA_BUNDLE_PATH: str | bool = True # use system default -else: - _CA_BUNDLE_PATH = _CA_BUNDLE - - -class FMPFundamentalProvider: - """Fetches fundamental data from Financial Modeling Prep REST API.""" - - def __init__(self, api_key: str) -> None: - if not api_key: - raise ProviderError("FMP API key is required") - self._api_key = api_key - - # Mapping from FMP endpoint name to the FundamentalData field it populates - _ENDPOINT_FIELD_MAP: dict[str, str] = { - "ratios-ttm": "pe_ratio", - "financial-growth": "revenue_growth", - "earnings": "earnings_surprise", - } - - async def fetch_fundamentals(self, ticker: str) -> FundamentalData: - """Fetch P/E, revenue growth, earnings surprise, and market cap. - - Fetches from multiple stable endpoints. If a supplementary endpoint - (ratios, growth, earnings) returns 402 (paid tier), we gracefully - degrade and return partial data rather than failing entirely, and - record the affected field in ``unavailable_fields``. - """ - try: - endpoints_402: set[str] = set() - - async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client: - params = {"symbol": ticker, "apikey": self._api_key} - - # Profile is the primary source — must succeed - profile = await self._fetch_json(client, "profile", params, ticker) - - # Supplementary sources — degrade gracefully on 402 - ratios, was_402 = await self._fetch_json_optional(client, "ratios-ttm", params, ticker) - if was_402: - endpoints_402.add("ratios-ttm") - - growth, was_402 = await self._fetch_json_optional(client, "financial-growth", params, ticker) - if was_402: - endpoints_402.add("financial-growth") - - earnings, was_402 = await self._fetch_json_optional(client, "earnings", params, ticker) - if was_402: - endpoints_402.add("earnings") - - pe_ratio = self._safe_float(ratios.get("priceToEarningsRatioTTM")) - revenue_growth = self._safe_float(growth.get("revenueGrowth")) - market_cap = self._safe_float(profile.get("marketCap")) - earnings_surprise = self._compute_earnings_surprise(earnings) - - # Build unavailable_fields from 402 endpoints - unavailable_fields: dict[str, str] = { - self._ENDPOINT_FIELD_MAP[ep]: "requires paid plan" - for ep in endpoints_402 - if ep in self._ENDPOINT_FIELD_MAP - } - - return FundamentalData( - ticker=ticker, - pe_ratio=pe_ratio, - revenue_growth=revenue_growth, - earnings_surprise=earnings_surprise, - market_cap=market_cap, - fetched_at=datetime.now(timezone.utc), - unavailable_fields=unavailable_fields, - ) - - except (ProviderError, RateLimitError): - raise - except Exception as exc: - logger.error("FMP provider error for %s: %s", ticker, exc) - raise ProviderError(f"FMP provider error for {ticker}: {exc}") from exc - - async def _fetch_json( - self, - client: httpx.AsyncClient, - endpoint: str, - params: dict, - ticker: str, - ) -> dict: - """Fetch a stable endpoint and return the first item (or empty dict).""" - url = f"{_FMP_STABLE_URL}/{endpoint}" - resp = await client.get(url, params=params) - self._check_response(resp, ticker, endpoint) - data = resp.json() - if isinstance(data, list): - return data[0] if data else {} - return data if isinstance(data, dict) else {} - - async def _fetch_json_optional( - self, - client: httpx.AsyncClient, - endpoint: str, - params: dict, - ticker: str, - ) -> tuple[dict, bool]: - """Fetch a stable endpoint, returning ``({}, True)`` on 402 (paid tier). - - Returns a tuple of (data_dict, was_402) so callers can track which - endpoints required a paid plan. - """ - url = f"{_FMP_STABLE_URL}/{endpoint}" - resp = await client.get(url, params=params) - if resp.status_code == 402: - logger.warning("FMP %s requires paid plan — skipping for %s", endpoint, ticker) - return {}, True - self._check_response(resp, ticker, endpoint) - data = resp.json() - if isinstance(data, list): - return (data[0] if data else {}, False) - return (data if isinstance(data, dict) else {}, False) - - def _compute_earnings_surprise(self, earnings_data: dict) -> float | None: - """Compute earnings surprise % from the most recent actual vs estimated EPS.""" - actual = self._safe_float(earnings_data.get("epsActual")) - estimated = self._safe_float(earnings_data.get("epsEstimated")) - if actual is None or estimated is None or estimated == 0: - return None - return ((actual - estimated) / abs(estimated)) * 100 - - def _check_response( - self, resp: httpx.Response, ticker: str, endpoint: str - ) -> None: - """Raise appropriate errors for non-200 responses.""" - if resp.status_code == 429: - raise RateLimitError(f"FMP rate limit hit for {ticker} ({endpoint})") - if resp.status_code == 403: - raise ProviderError( - f"FMP {endpoint} access denied for {ticker}: HTTP 403 — check API key validity and plan tier" - ) - if resp.status_code != 200: - raise ProviderError( - f"FMP {endpoint} error for {ticker}: HTTP {resp.status_code}" - ) - - @staticmethod - def _safe_float(value: object) -> float | None: - """Convert a value to float, returning None on failure.""" - if value is None: - return None - try: - return float(value) - except (TypeError, ValueError): - return None diff --git a/app/providers/fundamentals_chain.py b/app/providers/fundamentals_chain.py deleted file mode 100644 index cba5e85..0000000 --- a/app/providers/fundamentals_chain.py +++ /dev/null @@ -1,354 +0,0 @@ -"""Chained fundamentals provider with fallback adapters. - -Order: -1) FMP (if configured) -2) Finnhub (if configured) -3) Alpha Vantage (if configured) -""" - -from __future__ import annotations - -import logging -import os -from datetime import date, datetime, timedelta, timezone -from pathlib import Path - -import httpx - -from app.config import settings -from app.exceptions import ProviderError, RateLimitError -from app.providers.fmp import FMPFundamentalProvider -from app.providers.protocol import FundamentalData, FundamentalProvider - -logger = logging.getLogger(__name__) - -_CA_BUNDLE = os.environ.get("SSL_CERT_FILE", "") -if not _CA_BUNDLE or not Path(_CA_BUNDLE).exists(): - _CA_BUNDLE_PATH: str | bool = True -else: - _CA_BUNDLE_PATH = _CA_BUNDLE - - -def _safe_float(value: object) -> float | None: - if value is None: - return None - try: - return float(value) - except (TypeError, ValueError): - return None - - -def _to_api_symbol(symbol: str) -> str: - """Convert internal symbol format (BRK-B) to API format (BRK.B). - - Finnhub and Alpha Vantage use dot-separated share class notation. - """ - return symbol.replace("-", ".") - - -class FinnhubFundamentalProvider: - """Fundamentals provider backed by Finnhub free endpoints.""" - - def __init__(self, api_key: str) -> None: - if not api_key: - raise ProviderError("Finnhub API key is required") - self._api_key = api_key - self._base_url = "https://finnhub.io/api/v1" - - async def fetch_fundamentals(self, ticker: str) -> FundamentalData: - unavailable: dict[str, str] = {} - api_symbol = _to_api_symbol(ticker) - - today = date.today() - async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client: - profile_resp = await client.get( - f"{self._base_url}/stock/profile2", - params={"symbol": api_symbol, "token": self._api_key}, - ) - metric_resp = await client.get( - f"{self._base_url}/stock/metric", - params={"symbol": api_symbol, "metric": "all", "token": self._api_key}, - ) - earnings_resp = await client.get( - f"{self._base_url}/stock/earnings", - params={"symbol": api_symbol, "limit": 1, "token": self._api_key}, - ) - calendar_resp = await client.get( - f"{self._base_url}/calendar/earnings", - params={ - "symbol": api_symbol, - "from": today.isoformat(), - "to": (today + timedelta(days=120)).isoformat(), - "token": self._api_key, - }, - ) - - for resp, endpoint in ( - (profile_resp, "profile2"), - (metric_resp, "stock/metric"), - (earnings_resp, "stock/earnings"), - (calendar_resp, "calendar/earnings"), - ): - if resp.status_code == 429: - raise RateLimitError(f"Finnhub rate limit hit for {ticker} ({endpoint})") - if resp.status_code in (401, 403): - raise ProviderError(f"Finnhub access denied for {ticker} ({endpoint}): HTTP {resp.status_code}") - if resp.status_code != 200: - raise ProviderError(f"Finnhub error for {ticker} ({endpoint}): HTTP {resp.status_code}") - - profile_payload = profile_resp.json() if profile_resp.text else {} - metric_payload = metric_resp.json() if metric_resp.text else {} - earnings_payload = earnings_resp.json() if earnings_resp.text else [] - - metrics = metric_payload.get("metric", {}) if isinstance(metric_payload, dict) else {} - # Finnhub profile2 marketCapitalization is in millions of USD. - # Normalize to absolute dollars so cap bands / formatters match FMP & Alpha Vantage. - market_cap_millions = _safe_float((profile_payload or {}).get("marketCapitalization")) - market_cap = market_cap_millions * 1_000_000.0 if market_cap_millions is not None else None - pe_ratio = _safe_float(metrics.get("peTTM") or metrics.get("peNormalizedAnnual")) - revenue_growth = _safe_float(metrics.get("revenueGrowthTTMYoy") or metrics.get("revenueGrowth5Y")) - - earnings_surprise = None - if isinstance(earnings_payload, list) and earnings_payload: - first = earnings_payload[0] if isinstance(earnings_payload[0], dict) else {} - earnings_surprise = _safe_float(first.get("surprisePercent")) - - next_earnings_date = self._next_earnings(calendar_resp) - - if pe_ratio is None: - unavailable["pe_ratio"] = "not available from provider payload" - if revenue_growth is None: - unavailable["revenue_growth"] = "not available from provider payload" - if earnings_surprise is None: - unavailable["earnings_surprise"] = "not available from provider payload" - if market_cap is None: - unavailable["market_cap"] = "not available from provider payload" - - return FundamentalData( - ticker=ticker, - pe_ratio=pe_ratio, - revenue_growth=revenue_growth, - earnings_surprise=earnings_surprise, - market_cap=market_cap, - fetched_at=datetime.now(timezone.utc), - next_earnings_date=next_earnings_date, - unavailable_fields=unavailable, - ) - - @staticmethod - def _next_earnings(resp: httpx.Response) -> date | None: - """Earliest upcoming earnings date from Finnhub's calendar payload.""" - try: - payload = resp.json() if resp.text else {} - except ValueError: - return None - entries = payload.get("earningsCalendar", []) if isinstance(payload, dict) else [] - dates: list[date] = [] - today = date.today() - for entry in entries if isinstance(entries, list) else []: - raw = entry.get("date") if isinstance(entry, dict) else None - if not raw: - continue - try: - parsed = date.fromisoformat(raw) - except ValueError: - continue - if parsed >= today: - dates.append(parsed) - return min(dates) if dates else None - - -class AlphaVantageFundamentalProvider: - """Fundamentals provider backed by Alpha Vantage free endpoints.""" - - def __init__(self, api_key: str) -> None: - if not api_key: - raise ProviderError("Alpha Vantage API key is required") - self._api_key = api_key - self._base_url = "https://www.alphavantage.co/query" - - async def fetch_fundamentals(self, ticker: str) -> FundamentalData: - unavailable: dict[str, str] = {} - api_symbol = _to_api_symbol(ticker) - - async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client: - overview_resp = await client.get( - self._base_url, - params={"function": "OVERVIEW", "symbol": api_symbol, "apikey": self._api_key}, - ) - earnings_resp = await client.get( - self._base_url, - params={"function": "EARNINGS", "symbol": api_symbol, "apikey": self._api_key}, - ) - income_resp = await client.get( - self._base_url, - params={"function": "INCOME_STATEMENT", "symbol": api_symbol, "apikey": self._api_key}, - ) - - for resp, endpoint in ( - (overview_resp, "OVERVIEW"), - (earnings_resp, "EARNINGS"), - (income_resp, "INCOME_STATEMENT"), - ): - if resp.status_code == 429: - raise RateLimitError(f"Alpha Vantage rate limit hit for {ticker} ({endpoint})") - if resp.status_code != 200: - raise ProviderError(f"Alpha Vantage error for {ticker} ({endpoint}): HTTP {resp.status_code}") - - overview = overview_resp.json() if overview_resp.text else {} - earnings = earnings_resp.json() if earnings_resp.text else {} - income = income_resp.json() if income_resp.text else {} - - if isinstance(overview, dict) and overview.get("Information"): - raise ProviderError(f"Alpha Vantage unavailable for {ticker}: {overview.get('Information')}") - if isinstance(overview, dict) and overview.get("Note"): - raise RateLimitError(f"Alpha Vantage rate limit for {ticker}: {overview.get('Note')}") - - pe_ratio = _safe_float((overview or {}).get("PERatio")) - market_cap = _safe_float((overview or {}).get("MarketCapitalization")) - - earnings_surprise = None - quarterly = earnings.get("quarterlyEarnings", []) if isinstance(earnings, dict) else [] - if isinstance(quarterly, list) and quarterly: - first = quarterly[0] if isinstance(quarterly[0], dict) else {} - earnings_surprise = _safe_float(first.get("surprisePercentage")) - - revenue_growth = None - annual = income.get("annualReports", []) if isinstance(income, dict) else [] - if isinstance(annual, list) and len(annual) >= 2: - curr = _safe_float((annual[0] or {}).get("totalRevenue")) - prev = _safe_float((annual[1] or {}).get("totalRevenue")) - if curr is not None and prev not in (None, 0): - revenue_growth = ((curr - prev) / abs(prev)) * 100.0 - - if pe_ratio is None: - unavailable["pe_ratio"] = "not available from provider payload" - if revenue_growth is None: - unavailable["revenue_growth"] = "not available from provider payload" - if earnings_surprise is None: - unavailable["earnings_surprise"] = "not available from provider payload" - if market_cap is None: - unavailable["market_cap"] = "not available from provider payload" - - return FundamentalData( - ticker=ticker, - pe_ratio=pe_ratio, - revenue_growth=revenue_growth, - earnings_surprise=earnings_surprise, - market_cap=market_cap, - fetched_at=datetime.now(timezone.utc), - unavailable_fields=unavailable, - ) - - -_FUNDAMENTAL_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise", "market_cap") - - -class ChainedFundamentalProvider: - """Merge fundamentals across providers, filling gaps from later sources. - - A single provider rarely covers everything on free tiers — FMP's free plan, - for example, returns only market cap (the ratios/growth/earnings endpoints - 402). Rather than stop at the first provider with *any* field, we take each - field from the first provider that supplies it, so FMP's market cap is - combined with Finnhub's P/E and earnings surprise. - """ - - def __init__(self, providers: list[tuple[str, FundamentalProvider]]) -> None: - if not providers: - raise ProviderError("No fundamental providers configured") - self._providers = providers - - async def fetch_fundamentals(self, ticker: str, allow_partial: bool = False) -> FundamentalData: - """Merge fundamentals across providers. - - ``allow_partial`` controls behaviour when a fallback provider is *rate - limited* and we end up with missing fields. By default we raise - RateLimitError so the caller (the bulk collector) can back off and retry - the ticker once the window frees — otherwise a transient 429 on Finnhub - would be silently stored as market-cap-only. Pass ``allow_partial=True`` - (manual single fetches, or the collector's final give-up attempt) to - accept whatever was gathered instead of raising. - """ - merged: dict[str, float | None] = {f: None for f in _FUNDAMENTAL_FIELDS} - field_source: dict[str, str] = {} - errors: list[str] = [] - rate_limited = False - next_earnings_date = None - - for provider_name, provider in self._providers: - if all(merged[f] is not None for f in _FUNDAMENTAL_FIELDS) and next_earnings_date: - break - try: - data = await provider.fetch_fundamentals(ticker) - except RateLimitError as exc: - rate_limited = True - errors.append(f"{provider_name}: RateLimitError: {exc}") - continue - except Exception as exc: - errors.append(f"{provider_name}: {type(exc).__name__}: {exc}") - continue - - if next_earnings_date is None and data.next_earnings_date is not None: - next_earnings_date = data.next_earnings_date - - for field in _FUNDAMENTAL_FIELDS: - if merged[field] is None: - value = getattr(data, field) - if value is not None: - merged[field] = value - field_source[field] = provider_name - - missing = [f for f in _FUNDAMENTAL_FIELDS if merged[f] is None] - - # A rate limit left data incomplete: signal it (unless partial is OK) so - # the collector backs off rather than persisting a degraded record. - if rate_limited and missing and not allow_partial: - attempts = "; ".join(errors[:6]) - raise RateLimitError( - f"Fundamentals incomplete for {ticker} due to provider rate limits " - f"(missing {', '.join(missing)}). Attempts: {attempts}" - ) - - if all(merged[f] is None for f in _FUNDAMENTAL_FIELDS): - attempts = "; ".join(errors[:6]) if errors else "no usable metrics from any provider" - raise ProviderError(f"All fundamentals providers failed for {ticker}. Attempts: {attempts}") - - unavailable: dict[str, str] = { - field: "not available from any configured provider" - for field in _FUNDAMENTAL_FIELDS - if merged[field] is None - } - # Record which provider supplied each field for transparency. - for field, src in field_source.items(): - unavailable[f"source_{field}"] = src - - return FundamentalData( - ticker=ticker, - pe_ratio=merged["pe_ratio"], - revenue_growth=merged["revenue_growth"], - earnings_surprise=merged["earnings_surprise"], - market_cap=merged["market_cap"], - fetched_at=datetime.now(timezone.utc), - next_earnings_date=next_earnings_date, - unavailable_fields=unavailable, - ) - - -def build_fundamental_provider_chain() -> FundamentalProvider: - providers: list[tuple[str, FundamentalProvider]] = [] - - if settings.fmp_api_key: - providers.append(("fmp", FMPFundamentalProvider(settings.fmp_api_key))) - if settings.finnhub_api_key: - providers.append(("finnhub", FinnhubFundamentalProvider(settings.finnhub_api_key))) - if settings.alpha_vantage_api_key: - providers.append(("alpha_vantage", AlphaVantageFundamentalProvider(settings.alpha_vantage_api_key))) - - if not providers: - raise ProviderError( - "No fundamentals provider configured. Set one of FMP_API_KEY, FINNHUB_API_KEY, ALPHA_VANTAGE_API_KEY" - ) - - logger.info("Fundamentals provider chain configured: %s", [name for name, _ in providers]) - return ChainedFundamentalProvider(providers) diff --git a/app/providers/protocol.py b/app/providers/protocol.py index 9ce892f..b282d2f 100644 --- a/app/providers/protocol.py +++ b/app/providers/protocol.py @@ -44,20 +44,6 @@ class SentimentData: recommendation: str | None = None # "buy" | "hold" | "avoid" — actionable LLM view -@dataclass(frozen=True, slots=True) -class FundamentalData: - """Fundamental metrics returned by fundamental providers.""" - - ticker: str - pe_ratio: float | None - revenue_growth: float | None - earnings_surprise: float | None - market_cap: float | None - fetched_at: datetime - next_earnings_date: date | None = None - unavailable_fields: dict[str, str] = field(default_factory=dict) - - # --------------------------------------------------------------------------- # Provider Protocols # --------------------------------------------------------------------------- @@ -81,9 +67,5 @@ class SentimentProvider(Protocol): ... -class FundamentalProvider(Protocol): - """Protocol for fundamental data providers.""" - - async def fetch_fundamentals(self, ticker: str) -> FundamentalData: - """Fetch fundamental data for a ticker.""" - ... +# No fundamentals provider protocol: since A6 fundamentals come only from the +# batch SEC/Dolt imports, never from a request-time provider call. diff --git a/app/routers/admin.py b/app/routers/admin.py index c555f5e..73582f9 100644 --- a/app/routers/admin.py +++ b/app/routers/admin.py @@ -13,7 +13,6 @@ from app.schemas.admin import ( AlertConfigUpdate, CreateUserRequest, DataCleanupRequest, - FundamentalsCutoverConfigUpdate, JobTriggerRequest, JobToggle, RecommendationConfigUpdate, @@ -138,27 +137,6 @@ async def list_settings( ) -@router.get("/admin/settings/fundamentals-cutover", response_model=APIEnvelope) -async def get_fundamentals_cutover_settings( - _admin: User = Depends(require_admin), - db: AsyncSession = Depends(get_db), -): - config = await admin_service.get_fundamentals_cutover_config(db) - return APIEnvelope(status="success", data=config) - - -@router.put("/admin/settings/fundamentals-cutover", response_model=APIEnvelope) -async def update_fundamentals_cutover_settings( - body: FundamentalsCutoverConfigUpdate, - _admin: User = Depends(require_admin), - db: AsyncSession = Depends(get_db), -): - config = await admin_service.update_fundamentals_cutover_config( - db, body.enabled - ) - return APIEnvelope(status="success", data=config) - - @router.get("/admin/settings/recommendations", response_model=APIEnvelope) async def get_recommendation_settings( _admin: User = Depends(require_admin), @@ -475,36 +453,6 @@ async def toggle_job( ) -@router.get("/admin/fundamentals-parity", response_model=APIEnvelope) -async def get_fundamentals_parity_report( - _admin: User = Depends(require_admin), -): - """Latest read-only A5 source/score comparison, or null before first run.""" - return APIEnvelope( - status="success", data=admin_service.get_fundamentals_parity_report() - ) - - -@router.get("/admin/fundamentals-parity/csv", response_model=APIEnvelope) -async def get_fundamentals_parity_csv( - _admin: User = Depends(require_admin), -): - """Latest flattened A5 report for an authenticated browser download.""" - artifact = admin_service.get_fundamentals_parity_csv() - data = None if artifact is None else {"filename": artifact[0], "content": artifact[1]} - return APIEnvelope(status="success", data=data) - - -@router.get("/admin/fundamentals-parity/json", response_model=APIEnvelope) -async def get_fundamentals_parity_json( - _admin: User = Depends(require_admin), -): - """Canonical A5 JSON artifact for an authenticated browser download.""" - artifact = admin_service.get_fundamentals_parity_json() - data = None if artifact is None else {"filename": artifact[0], "content": artifact[1]} - return APIEnvelope(status="success", data=data) - - # --------------------------------------------------------------------------- # System events (operational warnings / errors) # --------------------------------------------------------------------------- diff --git a/app/routers/ingestion.py b/app/routers/ingestion.py index 32c29e9..a76056d 100644 --- a/app/routers/ingestion.py +++ b/app/routers/ingestion.py @@ -23,7 +23,6 @@ from app.models.sr_level import SRLevel from app.models.ticker import Ticker from app.models.user import User from app.providers.alpaca import AlpacaOHLCVProvider -from app.providers.fundamentals_chain import build_fundamental_provider_chain from app.services.rr_scanner_service import ( resolve_activation_ranks_for_symbol, scan_ticker, @@ -31,7 +30,6 @@ from app.services.rr_scanner_service import ( from app.services.sentiment_provider_service import build_sentiment_provider from app.schemas.common import APIEnvelope from app.services import ( - fundamental_service, ingestion_service, scoring_service, sentiment_service, @@ -185,34 +183,14 @@ async def fetch_symbol( sources_out["sentiment"] = {"status": "error", "message": str(exc)} # --- Fundamentals --- + # No per-ticker fetch exists any more: fundamental_data is rebuilt for the + # whole universe by the nightly SEC + Dolt imports, from local PostgreSQL. + # The source key is still accepted so older clients get a truthful answer. if "fundamentals" in requested: - if settings.fmp_api_key or settings.finnhub_api_key or settings.alpha_vantage_api_key: - try: - fundamentals_provider = build_fundamental_provider_chain() - # Manual single fetch: take whatever we can get (a lone 429 on a - # fallback shouldn't fail the whole refresh). - fdata = await fundamentals_provider.fetch_fundamentals( - symbol_upper, allow_partial=True - ) - await fundamental_service.store_fundamental( - db, - symbol=symbol_upper, - pe_ratio=fdata.pe_ratio, - revenue_growth=fdata.revenue_growth, - earnings_surprise=fdata.earnings_surprise, - market_cap=fdata.market_cap, - next_earnings_date=fdata.next_earnings_date, - unavailable_fields=fdata.unavailable_fields, - ) - sources_out["fundamentals"] = {"status": "ok", "message": None} - except Exception as exc: - logger.error("Fundamentals fetch failed for %s: %s", symbol_upper, exc) - sources_out["fundamentals"] = {"status": "error", "message": str(exc)} - else: - sources_out["fundamentals"] = { - "status": "skipped", - "message": "No fundamentals provider key configured", - } + sources_out["fundamentals"] = { + "status": "skipped", + "message": "Fundamentals refresh nightly from the SEC + Dolt imports", + } # --- Derived pipeline: S/R levels (free, always) --- try: diff --git a/app/scheduler.py b/app/scheduler.py index 68a13f7..d6ad8b4 100644 --- a/app/scheduler.py +++ b/app/scheduler.py @@ -1,9 +1,9 @@ """APScheduler job definitions and FastAPI lifespan integration. -Defines four scheduled jobs: +Defines the scheduled jobs, among them: - Data Collector (OHLCV fetch for all tickers) - Sentiment Collector (sentiment for all tickers) - - Fundamental Collector (fundamentals for all tickers) + - Dolt Earnings / SEC Fundamentals imports (bulk fundamentals sources) - R:R Scanner (trade setup scan for all tickers) Each job processes tickers independently, logs errors as structured JSON, @@ -25,22 +25,18 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.config import settings from app.database import async_session_factory -from app.models.fundamental import FundamentalData from app.models.ohlcv import OHLCVRecord from app.models.sentiment import SentimentScore from app.models.ticker import Ticker from app.exceptions import ProviderError from app.providers.alpaca import AlpacaOHLCVProvider -from app.providers.fundamentals_chain import build_fundamental_provider_chain from app.providers.protocol import SentimentData from app.services import ( - fundamental_service, ingestion_service, pipeline_run, sentiment_service, settings_store, shadow_book_service, - fundamentals_parity_service, fundamental_data_refresh_service, ) from app.services.data_import import ( @@ -93,7 +89,6 @@ _last_successful: dict[str, str | None] = { "data_collector": None, "data_backfill": None, "sentiment_collector": None, - "fundamental_collector": None, } # Jobs whose per-run progress is surfaced to Admin → Jobs. (outcome_evaluator is @@ -102,10 +97,8 @@ _JOB_NAMES = [ "data_collector", "data_backfill", "sentiment_collector", - "fundamental_collector", "dolt_earnings_import", "sec_fundamentals_import", - "fundamentals_parity_report", "rr_scanner", "ticker_universe_sync", "alerts", @@ -466,23 +459,6 @@ async def _get_sentiment_priority_tickers(db: AsyncSession) -> list[str]: return priority_syms + filler_syms -async def _get_fundamental_priority_tickers(db: AsyncSession) -> list[str]: - """Return symbols prioritized for fundamentals refresh. - - Priority: - 1) Tickers with no fundamentals snapshot yet - 2) Tickers with existing fundamentals, oldest fetched_at first - 3) Alphabetical tiebreaker - """ - missing_first = case((FundamentalData.fetched_at.is_(None), 0), else_=1) - result = await db.execute( - select(Ticker.symbol) - .outerjoin(FundamentalData, FundamentalData.ticker_id == Ticker.id) - .order_by(missing_first.asc(), FundamentalData.fetched_at.asc(), Ticker.symbol.asc()) - ) - return list(result.scalars().all()) - - def _resume_tickers(symbols: list[str], job_name: str) -> list[str]: """Reorder tickers to resume after the last successful one (rate-limit resume). @@ -815,139 +791,6 @@ async def collect_sentiment() -> None: _runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc)) -# --------------------------------------------------------------------------- -# Job: Fundamental Collector -# --------------------------------------------------------------------------- - - -async def collect_fundamentals() -> None: - """Fetch fundamentals for all tracked tickers via FMP. - - Processes each ticker independently. On rate limit, records last - successful ticker for resume. - """ - job_name = "fundamental_collector" - _log_event(logging.INFO, "job_start", job=job_name) - _runtime_start(job_name) - processed = 0 - total: int | None = None - - try: - async with async_session_factory() as db: - if not await _is_job_enabled(db, job_name): - _log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled") - _runtime_finish(job_name, "skipped", processed=0, total=0, message="Disabled") - return - if await fundamental_data_refresh_service.is_enabled(db): - message = "SEC + Dolt fundamentals cutover is active" - _log_event( - logging.INFO, - "job_skipped", - job=job_name, - reason="sec_dolt_cutover_active", - ) - _runtime_finish( - job_name, - "skipped", - processed=0, - total=0, - message=message, - ) - return - - symbols = await _get_fundamental_priority_tickers(db) - if not symbols: - _log_event(logging.INFO, "job_complete", job=job_name, tickers=0) - _runtime_finish(job_name, "completed", processed=0, total=0, message="No tickers") - return - - total = len(symbols) - _runtime_progress(job_name, processed=0, total=total) - - if not (settings.fmp_api_key or settings.finnhub_api_key or settings.alpha_vantage_api_key): - _log_event(logging.WARNING, "job_skipped", job=job_name, reason="no fundamentals provider keys configured") - _runtime_finish(job_name, "skipped", processed=0, total=total, message="No fundamentals provider keys configured") - return - - try: - provider = build_fundamental_provider_chain() - except Exception as exc: - _log_event(logging.ERROR, "job_error", job=job_name, error_type=type(exc).__name__, message=str(exc)) - _runtime_finish(job_name, "error", processed=0, total=total, message=str(exc)) - return - - max_retries = max(0, settings.fundamental_rate_limit_retries) - base_backoff = max(1, settings.fundamental_rate_limit_backoff_seconds) - spacing = max(0.0, settings.fundamental_request_spacing_seconds) - - async def _store(symbol: str, data) -> None: - async with async_session_factory() as db: - await fundamental_service.store_fundamental( - db, - symbol=symbol, - pe_ratio=data.pe_ratio, - revenue_growth=data.revenue_growth, - earnings_surprise=data.earnings_surprise, - market_cap=data.market_cap, - next_earnings_date=data.next_earnings_date, - unavailable_fields=data.unavailable_fields, - ) - - for symbol in symbols: - _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol) - attempt = 0 - while True: - try: - data = await provider.fetch_fundamentals(symbol) - await _store(symbol, data) - _last_successful[job_name] = symbol - processed += 1 - _runtime_progress(job_name, processed=processed, total=total, current_ticker=symbol) - _log_event(logging.INFO, "ticker_collected", job=job_name, ticker=symbol) - break - except Exception as exc: - msg = str(exc).lower() - if "rate" in msg or "429" in msg: - if attempt < max_retries: - wait_seconds = base_backoff * (2 ** attempt) - attempt += 1 - _log_event(logging.WARNING, "rate_limited_retry", job=job_name, ticker=symbol, attempt=attempt, max_retries=max_retries, wait_seconds=wait_seconds, processed=processed) - _runtime_progress( - job_name, - processed=processed, - total=total, - current_ticker=symbol, - message=f"Rate-limited at {symbol}; retry {attempt}/{max_retries} in {wait_seconds}s", - ) - await asyncio.sleep(wait_seconds) - continue - - # Retries exhausted: store whatever partial data we can - # still get (e.g. FMP market cap) and move on, rather than - # aborting the whole run and leaving every later ticker - # untouched. - _log_event(logging.WARNING, "rate_limited_partial", job=job_name, ticker=symbol, processed=processed) - try: - data = await provider.fetch_fundamentals(symbol, allow_partial=True) - await _store(symbol, data) - processed += 1 - except Exception as exc2: - _log_job_error(job_name, symbol, exc2) - break - _log_job_error(job_name, symbol, exc) - break - - if spacing: - await asyncio.sleep(spacing) - - _last_successful[job_name] = None - _log_event(logging.INFO, "job_complete", job=job_name, tickers=processed) - _runtime_finish(job_name, "completed", processed=processed, total=total, message=f"Processed {processed} tickers") - except Exception as exc: - _log_event(logging.ERROR, "job_error", job=job_name, error_type=type(exc).__name__, message=str(exc)) - _runtime_finish(job_name, "error", processed=processed, total=total, message=str(exc)) - - # --------------------------------------------------------------------------- # Jobs: shadow fundamentals sources # --------------------------------------------------------------------------- @@ -956,9 +799,9 @@ async def collect_fundamentals() -> None: async def _run_shadow_import(job_name: str, importer: SourceImporter) -> bool: """Run an importer and return whether its scheduled job was enabled. - The SEC wrapper uses the return value to run its activated local cache step - after deferred, failed, no-op, promoted, or source-locked attempts while honoring - the job-level disable switch. + The SEC wrapper uses the return value only to word its runtime message: its + local cache step runs after deferred, failed, no-op, promoted, source-locked + and disabled attempts alike. """ _log_event(logging.INFO, "job_start", job=job_name) _runtime_start(job_name, total=1) @@ -968,7 +811,7 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> bool: if not await _is_job_enabled(db, job_name): _log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled") _runtime_finish(job_name, "skipped", processed=0, total=1, message="Disabled") - return + return False run = await run_import(importer) if run is None: @@ -1015,25 +858,26 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> bool: async def run_dolt_earnings_import() -> None: - """Pull and import the Dolt earnings calendar/results feed in shadow.""" + """Pull and import the Dolt earnings calendar/results feed.""" await _run_shadow_import("dolt_earnings_import", DoltEarningsImporter()) async def run_sec_fundamentals_import() -> None: - """Import SEC facts, then run the activated local compat-cache refresh. + """Import SEC facts, then refresh the local compat cache. - The refresh is deliberately separate from the network import result. Once - activated it therefore still runs from stored snapshots/earnings/prices when - SEC is unavailable, unchanged, or another SEC import owns the source lock. + The refresh is deliberately independent of the network import: it reads only + stored snapshots, earnings events and closes, so it runs identically when SEC + is unavailable, unchanged, or owned by another import — and also when the + job's ingestion is switched off in Admin → Jobs. Disabling the job stops + SEC network access, not the cache; prices and earnings move daily even when + no filing does, and `fundamental_data` feeds scoring. """ job_name = "sec_fundamentals_import" - job_enabled = await _run_shadow_import(job_name, SecFundamentalsImporter()) - if not job_enabled: - return + import_ran = await _run_shadow_import(job_name, SecFundamentalsImporter()) try: async with async_session_factory() as db: - summary = await fundamental_data_refresh_service.refresh_if_enabled(db) + summary = await fundamental_data_refresh_service.refresh(db) except asyncio.CancelledError: _runtime_finish( job_name, "error", processed=0, total=1, message="Cancelled" @@ -1051,29 +895,27 @@ async def run_sec_fundamentals_import() -> None: _runtime_finish(job_name, "error", processed=0, total=1, message=message) return - if not summary["enabled"]: - _log_event( - logging.INFO, - "fundamental_data_refresh_skipped", - job=job_name, - reason="cutover_disabled", - setting=fundamental_data_refresh_service.ACTIVATION_KEY, - ) - return - _log_event( logging.INFO, "fundamental_data_refresh_complete", job=job_name, **summary, ) + cache_message = ( + f"cache {summary['refreshed']} · " + f"{summary['score_inputs_changed']} score inputs changed" + ) runtime = get_job_runtime_snapshot(job_name) - if runtime.get("status") == "completed": - import_message = runtime.get("message") or "import completed" - cache_message = ( - f"cache {summary['refreshed']} · " - f"{summary['score_inputs_changed']} score inputs changed" + if not import_ran: + _runtime_finish( + job_name, + "completed", + processed=1, + total=1, + message=f"Import disabled · {cache_message}", ) + elif runtime.get("status") == "completed": + import_message = runtime.get("message") or "import completed" _runtime_finish( job_name, "completed", @@ -1083,49 +925,6 @@ async def run_sec_fundamentals_import() -> None: ) -async def run_fundamentals_parity_report() -> None: - """Generate the A5 comparison bundle without mutating live fundamentals/scores.""" - job_name = "fundamentals_parity_report" - _log_event(logging.INFO, "job_start", job=job_name) - _runtime_start(job_name, total=1) - try: - async with async_session_factory() as db: - if not await _is_job_enabled(db, job_name): - _runtime_finish( - job_name, "skipped", processed=0, total=1, message="Disabled" - ) - return - report, artifacts = await fundamentals_parity_service.generate_and_store( - db, settings.fundamentals_parity_report_dir - ) - summary = report["summary"] - message = ( - f"{summary['universe_count']} tickers · " - f"{summary['fundamental_score_material_changes']} material score changes" - ) - _runtime_finish(job_name, "completed", processed=1, total=1, message=message) - _log_event( - logging.INFO, - "job_complete", - job=job_name, - generated_at=report["generated_at"], - json_path=artifacts["json"], - csv_path=artifacts["csv"], - ) - except asyncio.CancelledError: - _runtime_finish(job_name, "error", processed=0, total=1, message="Cancelled") - raise - except Exception as exc: - _runtime_finish(job_name, "error", processed=0, total=1, message=str(exc)) - _log_event( - logging.ERROR, - "job_error", - job=job_name, - error_type=type(exc).__name__, - message=str(exc), - ) - - # --------------------------------------------------------------------------- # Job: R:R Scanner # --------------------------------------------------------------------------- @@ -1666,19 +1465,16 @@ SCHEDULE_DEFAULTS: dict[str, str] = { "schedule_timezone": "America/New_York", # Morning data/display refresh (no qualifying R:R scan). "schedule_daily_pipeline_cron": "0 2 * * *", - # Bulk source imports. The SEC job writes the legacy compat cache only after - # the explicit, default-off A5 cutover setting is enabled. + # Bulk source imports. The SEC job also refreshes the fundamental_data compat + # cache that scoring reads — locally, from stored snapshots/earnings/closes. "schedule_dolt_earnings_cron": "30 2 * * *", "schedule_sec_fundamentals_cron": "0 4 * * *", - "schedule_fundamentals_parity_cron": "30 5 * * *", # Fetch in-progress bars → scan → Telegram (manual MOC window). "schedule_near_close_pipeline_cron": "30 15 * * mon-fri", # Fetch final bars → outcome eval (must not run on the partial near-close bar). "schedule_after_close_pipeline_cron": "45 16 * * mon-fri", # Hourly mid-session price + outcome (10:00–15:00 ET Mon–Fri). "schedule_intraday_pipeline_cron": "0 10-15 * * mon-fri", - # Weekly fundamentals early Monday NY. - "schedule_fundamentals_cron": "0 1 * * mon", } # job id -> schedule setting key @@ -1686,11 +1482,9 @@ _CRON_JOBS: dict[str, str] = { "daily_pipeline": "schedule_daily_pipeline_cron", "dolt_earnings_import": "schedule_dolt_earnings_cron", "sec_fundamentals_import": "schedule_sec_fundamentals_cron", - "fundamentals_parity_report": "schedule_fundamentals_parity_cron", "near_close_pipeline": "schedule_near_close_pipeline_cron", "after_close_pipeline": "schedule_after_close_pipeline_cron", "intraday_pipeline": "schedule_intraday_pipeline_cron", - "fundamental_collector": "schedule_fundamentals_cron", } @@ -1779,7 +1573,7 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None: "schedule_dolt_earnings_cron", ), id="dolt_earnings_import", - name="Dolt Earnings Import (shadow)", + name="Dolt Earnings Import", replace_existing=True, ) scheduler.add_job( @@ -1793,17 +1587,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None: name="SEC Fundamentals Import", replace_existing=True, ) - scheduler.add_job( - run_fundamentals_parity_report, - _cron_trigger( - cfg["schedule_fundamentals_parity_cron"], - tz, - "schedule_fundamentals_parity_cron", - ), - id="fundamentals_parity_report", - name="Fundamentals Parity Report (read-only)", - replace_existing=True, - ) scheduler.add_job( run_near_close_pipeline, _cron_trigger( @@ -1831,13 +1614,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None: _cron_trigger(cfg["schedule_intraday_pipeline_cron"], tz, "schedule_intraday_pipeline_cron"), id="intraday_pipeline", name="Intraday Pipeline", replace_existing=True, ) - # Fundamentals — quarterly-ish data; weekly by default (conserves API quota). - # Its own early cron so the slow, rate-limited fetch finishes before the day. - scheduler.add_job( - collect_fundamentals, - _cron_trigger(cfg["schedule_fundamentals_cron"], tz, "schedule_fundamentals_cron"), - id="fundamental_collector", name="Fundamental Collector", replace_existing=True, - ) # Independent interval jobs (own cadence, no ordering dependency) scheduler.add_job( @@ -1879,9 +1655,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None: }, dolt_earnings_import={"cron": cfg["schedule_dolt_earnings_cron"]}, sec_fundamentals_import={"cron": cfg["schedule_sec_fundamentals_cron"]}, - fundamentals_parity_report={ - "cron": cfg["schedule_fundamentals_parity_cron"] - }, near_close_pipeline={ "cron": cfg["schedule_near_close_pipeline_cron"], "steps": [name for name, _ in _NEAR_CLOSE_PIPELINE_STEPS], @@ -1894,7 +1667,6 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None: "cron": cfg["schedule_intraday_pipeline_cron"], "steps": [name for name, _ in _INTRADAY_PIPELINE_STEPS], }, - fundamental_collector={"cron": cfg["schedule_fundamentals_cron"]}, independent=["ticker_universe_sync", "backtest"], manual_only=["alerts", "data_backfill", "event_study"], ) diff --git a/app/schemas/admin.py b/app/schemas/admin.py index e86f941..05cf492 100644 --- a/app/schemas/admin.py +++ b/app/schemas/admin.py @@ -73,11 +73,6 @@ class ActivationConfigUpdate(BaseModel): exclude_neutral: bool | None = None -class FundamentalsCutoverConfigUpdate(BaseModel): - """Switch the legacy fundamentals cache from quota APIs to SEC/Dolt.""" - enabled: bool - - class ScheduleConfigUpdate(BaseModel): """Cron schedule for the pipelines + fundamentals. Crons are 5-field (min hour dom month dow); timezone is an IANA name (e.g. America/New_York).""" @@ -85,11 +80,9 @@ class ScheduleConfigUpdate(BaseModel): schedule_daily_pipeline_cron: str | None = Field(default=None, max_length=120) schedule_dolt_earnings_cron: str | None = Field(default=None, max_length=120) schedule_sec_fundamentals_cron: str | None = Field(default=None, max_length=120) - schedule_fundamentals_parity_cron: str | None = Field(default=None, max_length=120) schedule_near_close_pipeline_cron: str | None = Field(default=None, max_length=120) schedule_after_close_pipeline_cron: str | None = Field(default=None, max_length=120) schedule_intraday_pipeline_cron: str | None = Field(default=None, max_length=120) - schedule_fundamentals_cron: str | None = Field(default=None, max_length=120) class PerformanceConfigUpdate(BaseModel): diff --git a/app/services/admin_service.py b/app/services/admin_service.py index d614c5d..6de12b2 100644 --- a/app/services/admin_service.py +++ b/app/services/admin_service.py @@ -17,7 +17,7 @@ from app.models.settings import SystemSetting from app.models.ticker import Ticker from app.models.trade_setup import TradeSetup from app.models.user import User -from app.services import fundamental_data_refresh_service, settings_store +from app.services import settings_store logger = logging.getLogger(__name__) @@ -159,28 +159,6 @@ async def update_setting(db: AsyncSession, key: str, value: str) -> SystemSettin return setting -# --------------------------------------------------------------------------- -# Fundamentals source cutover -# --------------------------------------------------------------------------- - -async def get_fundamentals_cutover_config(db: AsyncSession) -> dict[str, bool]: - """Return the explicit A5 cache-cutover switch (default off).""" - return {"enabled": await fundamental_data_refresh_service.is_enabled(db)} - - -async def update_fundamentals_cutover_config( - db: AsyncSession, enabled: bool -) -> dict[str, bool]: - """Activate or pause SEC/Dolt writes to the legacy fundamentals cache.""" - await settings_store.upsert_setting( - db, - fundamental_data_refresh_service.ACTIVATION_KEY, - "true" if enabled else "false", - ) - await db.commit() - return await get_fundamentals_cutover_config(db) - - # --------------------------------------------------------------------------- # Activation thresholds # --------------------------------------------------------------------------- @@ -633,10 +611,8 @@ VALID_JOB_NAMES = { "data_backfill", "benchmark_collector", "sentiment_collector", - "fundamental_collector", "dolt_earnings_import", "sec_fundamentals_import", - "fundamentals_parity_report", "rr_scanner", "ticker_universe_sync", "outcome_evaluator", @@ -657,10 +633,8 @@ JOB_LABELS = { "data_backfill": "Data Backfill (deep history)", "benchmark_collector": "Benchmark Collector", "sentiment_collector": "Sentiment Collector", - "fundamental_collector": "Fundamental Collector", - "dolt_earnings_import": "Dolt Earnings Import (shadow)", + "dolt_earnings_import": "Dolt Earnings Import", "sec_fundamentals_import": "SEC Fundamentals Import", - "fundamentals_parity_report": "Fundamentals Parity Report (read-only)", "rr_scanner": "R:R Scanner", "ticker_universe_sync": "Ticker Universe Sync", "outcome_evaluator": "Outcome Evaluator", @@ -799,30 +773,3 @@ async def toggle_job(db: AsyncSession, job_name: str, enabled: bool) -> SystemSe key = f"job_{job_name}_enabled" return await update_setting(db, key, str(enabled).lower()) - - -def get_fundamentals_parity_report() -> dict | None: - """Return the latest compact A5 summary, if the job has run.""" - from app.config import settings - from app.services.fundamentals_parity_service import load_latest - - report = load_latest(settings.fundamentals_parity_report_dir) - if report is not None: - report.pop("rows", None) # full per-ticker data is download-only - return report - - -def get_fundamentals_parity_csv() -> tuple[str, str] | None: - """Return the latest A5 CSV filename and content for authenticated download.""" - from app.config import settings - from app.services.fundamentals_parity_service import load_latest_csv - - return load_latest_csv(settings.fundamentals_parity_report_dir) - - -def get_fundamentals_parity_json() -> tuple[str, str] | None: - """Return the canonical A5 JSON artifact for authenticated download.""" - from app.config import settings - from app.services.fundamentals_parity_service import load_latest_json - - return load_latest_json(settings.fundamentals_parity_report_dir) diff --git a/app/services/fundamental_data_refresh_service.py b/app/services/fundamental_data_refresh_service.py index a171e3a..5968ede 100644 --- a/app/services/fundamental_data_refresh_service.py +++ b/app/services/fundamental_data_refresh_service.py @@ -1,4 +1,7 @@ -"""A5 activation: refresh the legacy fundamentals cache from local bulk data.""" +"""Refresh the fundamentals compat cache from local SEC/Dolt bulk data. + +``fundamental_data`` is the table scoring reads. This is its only writer. +""" from __future__ import annotations @@ -12,38 +15,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.database import insert_for_session from app.models.fundamental import FundamentalData from app.models.score import CompositeScore, DimensionScore -from app.services import fundamentals_candidate_service, settings_store +from app.services import fundamentals_candidate_service - -# Absence is deliberately false. Production activation therefore requires one -# explicit, durable SystemSetting change after the A5 evidence is approved. -ACTIVATION_KEY = "fundamental_data_sec_dolt_cutover_enabled" _SCORE_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise") -async def is_enabled(db: AsyncSession) -> bool: - raw = await settings_store.get_value(db, ACTIVATION_KEY, "false") - return str(raw).strip().lower() == "true" - - -async def refresh_if_enabled( - db: AsyncSession, - *, - now: datetime | None = None, - today: date | None = None, -) -> dict[str, Any]: - """Refresh atomically when activated; otherwise perform no writes.""" - if not await is_enabled(db): - return { - "enabled": False, - "refreshed": 0, - "score_inputs_changed": 0, - "dimension_scores_staled": 0, - "composite_scores_staled": 0, - } - return await refresh(db, now=now, today=today) - - async def refresh( db: AsyncSession, *, @@ -117,7 +93,6 @@ async def refresh( await db.commit() return { - "enabled": True, "refreshed": len(candidates), "score_inputs_changed": len(changed_ids), "dimension_scores_staled": len(dimension_ids), diff --git a/app/services/fundamental_service.py b/app/services/fundamental_service.py index 234d249..c6fcb65 100644 --- a/app/services/fundamental_service.py +++ b/app/services/fundamental_service.py @@ -1,22 +1,19 @@ -"""Fundamental data service. +"""Fundamental data read access. -Stores fundamental data (P/E, revenue growth, earnings surprise, market cap) -and marks the fundamental dimension score as stale on new data. +``fundamental_data`` is the compat cache scoring reads. It is written solely by +``fundamental_data_refresh_service`` from SEC snapshots, Dolt earnings events and +stored closes; nothing fetches it per ticker. """ from __future__ import annotations -import json import logging -from datetime import datetime, timezone -from sqlalchemy import select, update +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from app.database import insert_for_session from app.exceptions import NotFoundError from app.models.fundamental import FundamentalData -from app.models.score import DimensionScore from app.models.ticker import Ticker logger = logging.getLogger(__name__) @@ -32,65 +29,6 @@ async def _get_ticker(db: AsyncSession, symbol: str) -> Ticker: return ticker -async def store_fundamental( - db: AsyncSession, - symbol: str, - pe_ratio: float | None = None, - revenue_growth: float | None = None, - earnings_surprise: float | None = None, - market_cap: float | None = None, - next_earnings_date=None, - unavailable_fields: dict[str, str] | None = None, -) -> FundamentalData: - """Store or update fundamental data for a ticker. - - Keeps a single latest snapshot per ticker. On new data, marks the - fundamental dimension score as stale (if one exists). - """ - ticker = await _get_ticker(db, symbol) - - now = datetime.now(timezone.utc) - unavailable_fields_json = json.dumps(unavailable_fields or {}) - - stmt = insert_for_session(db, FundamentalData).values( - ticker_id=ticker.id, - pe_ratio=pe_ratio, - revenue_growth=revenue_growth, - earnings_surprise=earnings_surprise, - market_cap=market_cap, - next_earnings_date=next_earnings_date, - fetched_at=now, - unavailable_fields_json=unavailable_fields_json, - ) - stmt = stmt.on_conflict_do_update( - index_elements=["ticker_id"], - set_={ - "pe_ratio": stmt.excluded.pe_ratio, - "revenue_growth": stmt.excluded.revenue_growth, - "earnings_surprise": stmt.excluded.earnings_surprise, - "market_cap": stmt.excluded.market_cap, - "next_earnings_date": stmt.excluded.next_earnings_date, - "fetched_at": stmt.excluded.fetched_at, - "unavailable_fields_json": stmt.excluded.unavailable_fields_json, - }, - ).returning(FundamentalData) - record = (await db.execute(stmt)).scalar_one() - - # Mark fundamental dimension score as stale if it exists - # TODO: Use DimensionScore service when built - await db.execute( - update(DimensionScore) - .where( - DimensionScore.ticker_id == ticker.id, - DimensionScore.dimension == "fundamental", - ) - .values(is_stale=True) - ) - - await db.commit() - return record - - async def get_fundamental( db: AsyncSession, symbol: str, diff --git a/app/services/fundamentals_candidate_service.py b/app/services/fundamentals_candidate_service.py index 03d61f1..6ad0585 100644 --- a/app/services/fundamentals_candidate_service.py +++ b/app/services/fundamentals_candidate_service.py @@ -1,9 +1,8 @@ -"""Local SEC/Dolt candidate values for the legacy fundamentals cache. +"""Local SEC/Dolt candidate values for the fundamentals compat cache. -This is the single read path shared by the A5 parity report and the activated -``fundamental_data`` refresh. It never contacts SEC or Dolt: every input comes -from PostgreSQL, so price- and earnings-driven values can still refresh when an -upstream import is unchanged or unavailable. +This is the read path behind the ``fundamental_data`` refresh. It never contacts +SEC or Dolt: every input comes from PostgreSQL, so price- and earnings-driven +values can still refresh when an upstream import is unchanged or unavailable. """ from __future__ import annotations diff --git a/app/services/fundamentals_parity_service.py b/app/services/fundamentals_parity_service.py deleted file mode 100644 index de43060..0000000 --- a/app/services/fundamentals_parity_service.py +++ /dev/null @@ -1,498 +0,0 @@ -"""Read-only A5 comparison of legacy and SEC/Dolt fundamental inputs. - -The report deliberately does not write ``fundamental_data`` or score tables. -It reconstructs the current legacy and candidate fundamental scores, projects -their composite-score/rank effect with the active weights, and archives a -timestamped JSON + CSV bundle for explicit human approval. -""" - -from __future__ import annotations - -import csv -import io -import json -import math -import os -import statistics -from datetime import date, datetime, timezone -from pathlib import Path -from typing import Any, Iterable -from zoneinfo import ZoneInfo - -from sqlalchemy import select, text -from sqlalchemy.ext.asyncio import AsyncSession - -from app.models.data_import_run import DataImportRun -from app.models.fundamental import FundamentalData -from app.services import fundamentals_candidate_service as candidate_service - -REPORT_VERSION = 1 -APPROVAL_STATUS = "pending_explicit_approval" -FIELD_KEYS = ("pe_ratio", "revenue_growth", "earnings_surprise") -MIN_SCORE_METRICS = 2 - -# Materiality is a review aid, never an automatic cutover verdict. Definition -# changes remain visible even when a delta falls inside these bands. -FIELD_TOLERANCES = { - "pe_ratio": {"absolute": 1.0, "relative_pct": 10.0}, - "revenue_growth": {"absolute": 2.0, "relative_pct": None}, - "earnings_surprise": {"absolute": 2.0, "relative_pct": None}, -} -DEFINITION_NOTES = { - "pe_ratio": ( - "Legacy provider P/E convention versus latest close divided by " - "SEC-derived TTM diluted EPS." - ), - "revenue_growth": ( - "Legacy provider growth convention versus SEC-derived TTM revenue YoY." - ), - "earnings_surprise": ( - "Legacy provider latest surprise versus latest completed Dolt earnings " - "event with actual and estimate." - ), -} - - -def fundamental_score( - pe_ratio: float | None, - revenue_growth: float | None, - earnings_surprise: float | None, -) -> float | None: - """Match the production fundamental-dimension formula without persistence.""" - scores: list[float] = [] - if _finite(pe_ratio) and pe_ratio > 0: - scores.append(max(0.0, min(100.0, 100.0 - (pe_ratio - 15.0) * (100.0 / 30.0)))) - if _finite(revenue_growth): - scores.append(max(0.0, min(100.0, 50.0 + revenue_growth * 2.5))) - if _finite(earnings_surprise): - scores.append(max(0.0, min(100.0, 50.0 + earnings_surprise * 5.0))) - return sum(scores) / len(scores) if len(scores) >= MIN_SCORE_METRICS else None - - -async def build_report( - db: AsyncSession, - *, - generated_at: datetime | None = None, - today: date | None = None, -) -> dict[str, Any]: - """Build a point-in-time parity report from one database session.""" - generated_at = generated_at or datetime.now(timezone.utc) - today = today or datetime.now(ZoneInfo("America/New_York")).date() - - # A report must not mix rows from before and after a concurrent import - # promotion. The scheduled job provides a fresh session, so establish the - # production snapshot before its first query and have Postgres enforce the - # no-write contract as well. SQLite tests retain their normal transaction. - if db.get_bind().dialect.name == "postgresql": - connection = await db.connection( - execution_options={"isolation_level": "REPEATABLE READ"} - ) - await connection.execute(text("SET TRANSACTION READ ONLY")) - - candidates = await candidate_service.build_candidates(db, today=today) - ticker_ids = [candidate.ticker_id for candidate in candidates] - legacy_by_ticker = await _legacy_values(db, ticker_ids) - source_runs = await _source_runs(db) - - rows: list[dict[str, Any]] = [] - for candidate in candidates: - legacy = legacy_by_ticker.get(candidate.ticker_id) - candidate_values = { - "pe_ratio": candidate.pe_ratio, - "revenue_growth": candidate.revenue_growth, - "earnings_surprise": candidate.earnings_surprise, - } - legacy_values = { - "pe_ratio": legacy.pe_ratio if legacy else None, - "revenue_growth": legacy.revenue_growth if legacy else None, - "earnings_surprise": legacy.earnings_surprise if legacy else None, - } - fields = { - key: _field_comparison(key, legacy_values[key], candidate_values[key]) - for key in FIELD_KEYS - } - legacy_score = fundamental_score(**legacy_values) - candidate_score = fundamental_score(**candidate_values) - rows.append( - { - "symbol": candidate.symbol, - "cik": candidate.cik, - "legacy_fetched_at": _iso(legacy.fetched_at) if legacy else None, - "price_date": _iso(candidate.price_date), - "fields": fields, - "scores": { - "legacy_fundamental": _round(legacy_score), - "candidate_fundamental": _round(candidate_score), - "fundamental_delta": _delta(legacy_score, candidate_score), - "legacy_fundamental_rank": None, - "candidate_fundamental_rank": None, - "fundamental_rank_change": None, - }, - } - ) - - _attach_ranks(rows, "legacy_fundamental", "legacy_fundamental_rank") - _attach_ranks(rows, "candidate_fundamental", "candidate_fundamental_rank") - for row in rows: - scores = row["scores"] - scores["fundamental_rank_change"] = _rank_change( - scores["legacy_fundamental_rank"], scores["candidate_fundamental_rank"] - ) - - return { - "report_version": REPORT_VERSION, - "generated_at": generated_at.isoformat(), - "as_of_date": today.isoformat(), - "approval_status": APPROVAL_STATUS, - "read_only": True, - "fundamental_score_formula": ( - "Equal-weighted mean of 2+ available sub-scores: P/E = " - "clamp(100-(pe-15)*(100/30)); revenue growth = " - "clamp(50+growth*2.5); earnings surprise = " - "clamp(50+surprise*5)." - ), - "source_runs": source_runs, - "definition_notes": DEFINITION_NOTES, - "materiality_notes": { - "fields": FIELD_TOLERANCES, - "fundamental_score_absolute": 5.0, - "automatic_cutover": False, - }, - "summary": _summary(rows), - "rows": rows, - } - - -def store_report(report: dict[str, Any], report_dir: str | Path) -> dict[str, str]: - """Atomically archive JSON/CSV artifacts and update the latest manifest.""" - directory = Path(report_dir).expanduser().resolve() - directory.mkdir(parents=True, exist_ok=True) - stamp = _artifact_stamp(report["generated_at"]) - json_name = f"fundamentals-parity-{stamp}.json" - csv_name = f"fundamentals-parity-{stamp}.csv" - json_path = directory / json_name - csv_path = directory / csv_name - - _atomic_write(json_path, json.dumps(report, indent=2, sort_keys=True) + "\n") - _atomic_write(csv_path, report_csv(report)) - manifest = { - "generated_at": report["generated_at"], - "json_file": json_name, - "csv_file": csv_name, - } - _atomic_write( - directory / "latest.json", - json.dumps(manifest, indent=2, sort_keys=True) + "\n", - ) - return { - "json": str(json_path), - "csv": str(csv_path), - "manifest": str(directory / "latest.json"), - } - - -async def generate_and_store( - db: AsyncSession, - report_dir: str | Path, - *, - generated_at: datetime | None = None, - today: date | None = None, -) -> tuple[dict[str, Any], dict[str, str]]: - report = await build_report(db, generated_at=generated_at, today=today) - return report, store_report(report, report_dir) - - -def load_latest(report_dir: str | Path) -> dict[str, Any] | None: - manifest = _load_manifest(report_dir) - if manifest is None: - return None - try: - path = _manifest_artifact(report_dir, manifest, "json_file") - loaded = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError, TypeError, ValueError): - return None - return loaded if isinstance(loaded, dict) else None - - -def load_latest_csv(report_dir: str | Path) -> tuple[str, str] | None: - return _load_latest_text_artifact(report_dir, "csv_file") - - -def load_latest_json(report_dir: str | Path) -> tuple[str, str] | None: - return _load_latest_text_artifact(report_dir, "json_file") - - -def _load_latest_text_artifact( - report_dir: str | Path, manifest_key: str -) -> tuple[str, str] | None: - manifest = _load_manifest(report_dir) - if manifest is None: - return None - try: - path = _manifest_artifact(report_dir, manifest, manifest_key) - return path.name, path.read_text(encoding="utf-8") - except (OSError, TypeError, ValueError): - return None - - -def report_csv(report: dict[str, Any]) -> str: - output = io.StringIO(newline="") - columns = [ - "symbol", - "cik", - "legacy_fetched_at", - "price_date", - *( - f"{field}_{suffix}" - for field in FIELD_KEYS - for suffix in ("legacy", "candidate", "absolute_delta", "relative_delta_pct", "material") - ), - "legacy_fundamental", - "candidate_fundamental", - "fundamental_delta", - "legacy_fundamental_rank", - "candidate_fundamental_rank", - "fundamental_rank_change", - ] - writer = csv.DictWriter(output, fieldnames=columns) - writer.writeheader() - for row in report.get("rows", []): - flat = { - "symbol": row["symbol"], - "cik": row.get("cik"), - "legacy_fetched_at": row.get("legacy_fetched_at"), - "price_date": row.get("price_date"), - **row["scores"], - } - for field in FIELD_KEYS: - comparison = row["fields"][field] - for suffix in ( - "legacy", - "candidate", - "absolute_delta", - "relative_delta_pct", - "material", - ): - flat[f"{field}_{suffix}"] = comparison.get(suffix) - writer.writerow(flat) - return output.getvalue() - - -async def _legacy_values( - db: AsyncSession, ticker_ids: list[int] -) -> dict[int, FundamentalData]: - if not ticker_ids: - return {} - rows = ( - await db.execute( - select(FundamentalData).where(FundamentalData.ticker_id.in_(ticker_ids)) - ) - ).scalars() - return {row.ticker_id: row for row in rows} - - -async def _source_runs(db: AsyncSession) -> dict[str, dict[str, Any] | None]: - sources = ("sec_facts", "dolt_earnings") - rows = ( - await db.execute( - select(DataImportRun) - .where( - DataImportRun.source.in_(sources), - DataImportRun.status.in_(("promoted", "no_op")), - ) - .order_by(DataImportRun.id.desc()) - ) - ).scalars() - latest: dict[str, dict[str, Any] | None] = {source: None for source in sources} - for row in rows: - if latest[row.source] is None: - latest[row.source] = { - "run_id": row.id, - "status": row.status, - "revision": row.revision, - "source_max_date": _iso(row.source_max_date), - "completed_at": _iso(row.completed_at), - } - return latest - - -def _field_comparison( - key: str, legacy: float | None, candidate: float | None -) -> dict[str, Any]: - legacy = float(legacy) if _finite(legacy) else None - candidate = float(candidate) if _finite(candidate) else None - absolute = _delta(legacy, candidate) - relative = ( - None - if absolute is None or legacy in (None, 0) - else round(absolute / abs(legacy) * 100.0, 4) - ) - tolerance = FIELD_TOLERANCES[key] - material = False - if absolute is not None: - material = abs(absolute) > tolerance["absolute"] - relative_limit = tolerance["relative_pct"] - if relative_limit is not None: - material = material and relative is not None and abs(relative) > relative_limit - return { - "legacy": _round(legacy), - "candidate": _round(candidate), - "absolute_delta": absolute, - "relative_delta_pct": relative, - "material": material, - "definition_changed": True, - } - - -def _attach_ranks(rows: list[dict[str, Any]], value_key: str, rank_key: str) -> None: - values = [ - row["scores"][value_key] - for row in rows - if _finite(row["scores"][value_key]) - ] - for row in rows: - value = row["scores"][value_key] - row["scores"][rank_key] = ( - 1 + sum(other > value for other in values) if _finite(value) else None - ) - - -def _summary(rows: list[dict[str, Any]]) -> dict[str, Any]: - field_stats = {} - for key in FIELD_KEYS: - comparisons = [row["fields"][key] for row in rows] - deltas = [ - abs(item["absolute_delta"]) - for item in comparisons - if item["absolute_delta"] is not None - ] - field_stats[key] = { - "legacy_available": sum(item["legacy"] is not None for item in comparisons), - "candidate_available": sum( - item["candidate"] is not None for item in comparisons - ), - "both_available": len(deltas), - "material_differences": sum(item["material"] for item in comparisons), - "median_absolute_delta": _round(statistics.median(deltas) if deltas else None), - "p95_absolute_delta": _round(_percentile(deltas, 0.95)), - "max_absolute_delta": _round(max(deltas) if deltas else None), - } - - fundamental_deltas = _score_deltas(rows, "fundamental_delta") - changed_rows = sorted( - ( - { - "symbol": row["symbol"], - "fundamental_delta": row["scores"]["fundamental_delta"], - "fundamental_rank_change": row["scores"]["fundamental_rank_change"], - } - for row in rows - if row["scores"]["fundamental_delta"] is not None - ), - key=lambda item: ( - abs(item["fundamental_delta"] or 0), - ), - reverse=True, - )[:20] - return { - "universe_count": len(rows), - "legacy_fundamental_score_available": _count_score( - rows, "legacy_fundamental" - ), - "candidate_fundamental_score_available": _count_score( - rows, "candidate_fundamental" - ), - "fundamental_scores_compared": len(fundamental_deltas), - "fundamental_score_material_changes": sum( - abs(delta) > 5.0 for delta in fundamental_deltas - ), - "fundamental_rank_changes": _rank_change_count( - rows, "fundamental_rank_change" - ), - "field_stats": field_stats, - "largest_changes": changed_rows, - } - - -def _score_deltas(rows: Iterable[dict[str, Any]], key: str) -> list[float]: - return [ - row["scores"][key] - for row in rows - if row["scores"][key] is not None - ] - - -def _count_score(rows: Iterable[dict[str, Any]], key: str) -> int: - return sum(row["scores"][key] is not None for row in rows) - - -def _rank_change_count(rows: Iterable[dict[str, Any]], key: str) -> int: - return sum( - row["scores"][key] not in (None, 0) - for row in rows - ) - - -def _rank_change(legacy: int | None, candidate: int | None) -> int | None: - # Positive means the candidate improved its rank. - return legacy - candidate if legacy is not None and candidate is not None else None - - -def _delta(legacy: float | None, candidate: float | None) -> float | None: - if not _finite(legacy) or not _finite(candidate): - return None - return round(candidate - legacy, 4) - - -def _round(value: float | None, digits: int = 4) -> float | None: - return round(float(value), digits) if _finite(value) else None - - -def _percentile(values: list[float], quantile: float) -> float | None: - if not values: - return None - ordered = sorted(values) - index = max(0, math.ceil(quantile * len(ordered)) - 1) - return ordered[index] - - -def _finite(value: Any) -> bool: - return ( - isinstance(value, (int, float)) - and not isinstance(value, bool) - and math.isfinite(value) - ) - - -def _iso(value: Any) -> str | None: - return value.isoformat() if value is not None else None - - -def _artifact_stamp(raw: str) -> str: - parsed = datetime.fromisoformat(raw.replace("Z", "+00:00")) - return parsed.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ") - - -def _atomic_write(path: Path, content: str) -> None: - temp = path.with_name(f".{path.name}.{os.getpid()}.tmp") - temp.write_text(content, encoding="utf-8", newline="") - os.replace(temp, path) - - -def _load_manifest(report_dir: str | Path) -> dict[str, Any] | None: - path = Path(report_dir).expanduser().resolve() / "latest.json" - try: - loaded = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError, TypeError, ValueError): - return None - return loaded if isinstance(loaded, dict) else None - - -def _manifest_artifact( - report_dir: str | Path, manifest: dict[str, Any], key: str -) -> Path: - directory = Path(report_dir).expanduser().resolve() - name = Path(str(manifest.get(key, ""))).name - if not name: - raise ValueError(f"Latest parity manifest has no {key}") - return directory / name diff --git a/app/services/fundamentals_quality_service.py b/app/services/fundamentals_quality_service.py index aa76600..161e025 100644 --- a/app/services/fundamentals_quality_service.py +++ b/app/services/fundamentals_quality_service.py @@ -12,7 +12,6 @@ from app.models.data_import_run import DataImportRun from app.models.fundamental_snapshot import FundamentalSnapshot from app.models.sec_filing_gap import SecFilingGap from app.models.ticker import Ticker -from app.services import fundamental_data_refresh_service _SEC_FORMS = ("10-K", "10-Q", "10-K/A", "10-Q/A") @@ -78,8 +77,6 @@ async def blocked_reasons_by_cik( ciks: set[str] | None = None, ) -> dict[str, str]: """Current SEC blocker code by CIK; no historical audit scan.""" - if not await fundamental_data_refresh_service.is_enabled(db): - return {} if ciks is not None and not ciks: return {} diff --git a/app/services/scoring_service.py b/app/services/scoring_service.py index a6b3e67..90b01c0 100644 --- a/app/services/scoring_service.py +++ b/app/services/scoring_service.py @@ -497,8 +497,8 @@ async def _compute_fundamental_score( "reason": "Earnings surprise data not available", }) - # Require at least two real metrics — a single available metric (e.g. only - # market cap is free on FMP) does not make a meaningful fundamental score. + # Require at least two real metrics — a single available metric (e.g. an + # issuer with only a market cap) does not make a meaningful fundamental score. MIN_METRICS = 2 if len(scores) < MIN_METRICS: unavailable.append({ diff --git a/app/services/sec_client.py b/app/services/sec_client.py index ffedf24..1f68bdd 100644 --- a/app/services/sec_client.py +++ b/app/services/sec_client.py @@ -39,7 +39,7 @@ logger = logging.getLogger(__name__) _WWW = "https://www.sec.gov" _DATA = "https://data.sec.gov" -# Resolve CA bundle for explicit httpx verify (matches app/providers/fmp.py). +# Resolve CA bundle for explicit httpx verify (matches app/providers/alpaca.py). _CA = os.environ.get("SSL_CERT_FILE", "") _CA_VERIFY: str | bool = _CA if _CA and Path(_CA).exists() else True diff --git a/app/services/ticker_universe_service.py b/app/services/ticker_universe_service.py index fa1a0c9..b895c2a 100644 --- a/app/services/ticker_universe_service.py +++ b/app/services/ticker_universe_service.py @@ -113,116 +113,6 @@ def _normalise_symbols(symbols: Iterable[str]) -> list[str]: return sorted(deduped) -def _extract_symbols_from_fmp_payload(payload: object) -> list[str]: - if not isinstance(payload, list): - return [] - - symbols: list[str] = [] - for item in payload: - if not isinstance(item, dict): - continue - candidate = item.get("symbol") or item.get("ticker") - if isinstance(candidate, str): - symbols.append(candidate) - return symbols - - -async def _try_fmp_urls( - client: httpx.AsyncClient, - urls: list[str], -) -> tuple[list[str], list[str]]: - failures: list[str] = [] - for url in urls: - endpoint = url.split("?")[0] - try: - response = await client.get(url) - except httpx.HTTPError as exc: - failures.append(f"{endpoint}: network error ({type(exc).__name__}: {exc})") - continue - - if response.status_code != 200: - failures.append(f"{endpoint}: HTTP {response.status_code}") - continue - - try: - payload = response.json() - except ValueError: - failures.append(f"{endpoint}: invalid JSON payload") - continue - - symbols = _extract_symbols_from_fmp_payload(payload) - if symbols: - return symbols, failures - - failures.append(f"{endpoint}: empty/unsupported payload") - - return [], failures - - -async def _fetch_universe_symbols_from_fmp(universe: str) -> list[str]: - if not settings.fmp_api_key: - raise ValidationError( - "FMP API key is required for universe bootstrap (set FMP_API_KEY)" - ) - - api_key = settings.fmp_api_key - stable_base = "https://financialmodelingprep.com/stable" - legacy_base = "https://financialmodelingprep.com/api/v3" - - stable_candidates: dict[str, list[str]] = { - "sp500": [ - f"{stable_base}/sp500-constituent?apikey={api_key}", - f"{stable_base}/sp500-constituents?apikey={api_key}", - ], - "nasdaq100": [ - f"{stable_base}/nasdaq-100-constituent?apikey={api_key}", - f"{stable_base}/nasdaq100-constituent?apikey={api_key}", - f"{stable_base}/nasdaq-100-constituents?apikey={api_key}", - ], - "nasdaq_all": [ - f"{stable_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}", - f"{stable_base}/available-traded/list?apikey={api_key}", - ], - } - - legacy_candidates: dict[str, list[str]] = { - "sp500": [ - f"{legacy_base}/sp500_constituent?apikey={api_key}", - f"{legacy_base}/sp500_constituent", - ], - "nasdaq100": [ - f"{legacy_base}/nasdaq_constituent?apikey={api_key}", - f"{legacy_base}/nasdaq_constituent", - ], - "nasdaq_all": [ - f"{legacy_base}/stock-screener?exchange=NASDAQ&isEtf=false&limit=10000&apikey={api_key}", - ], - } - - failures: list[str] = [] - async with httpx.AsyncClient(timeout=30.0, verify=_CA_BUNDLE_PATH) as client: - stable_symbols, stable_failures = await _try_fmp_urls(client, stable_candidates[universe]) - failures.extend(stable_failures) - - if stable_symbols: - return stable_symbols - - legacy_symbols, legacy_failures = await _try_fmp_urls(client, legacy_candidates[universe]) - failures.extend(legacy_failures) - - if legacy_symbols: - return legacy_symbols - - if failures: - reason = "; ".join(failures[:6]) - logger.warning("FMP universe fetch failed for %s: %s", universe, reason) - raise ProviderError( - f"Failed to fetch universe symbols from FMP for '{universe}'. Attempts: {reason}" - ) - - raise ProviderError(f"Failed to fetch universe symbols from FMP for '{universe}'") - - async def _fetch_wiki_constituent_symbols( client: httpx.AsyncClient, url: str, @@ -351,13 +241,16 @@ async def fetch_universe_symbols( Fallback order: 1) Free public sources (Wikipedia/NASDAQ trader) - 2) FMP endpoints (if available) - 3) Cached snapshot in SystemSetting - 4) Built-in seed symbols + 2) Cached snapshot in SystemSetting + 3) Built-in seed symbols Returns ``(symbols, source_label)`` so bootstrap UI can show where the - list came from (important when Wikipedia/FMP fail and a stale cache still - lists BK instead of BNY). + list came from (important when the public source fails and a stale cache + still lists BK instead of BNY). + + The seeds are representative, not complete, so a *fresh* install whose + public source is down bootstraps a partial universe. A warm instance is + unaffected — it falls through to its cached snapshot. """ normalised_universe = _validate_universe(universe) failures: list[str] = [] @@ -369,15 +262,6 @@ async def fetch_universe_symbols( await _write_cached_symbols(db, normalised_universe, cleaned_public, public_source or "public") return cleaned_public, public_source or "public" - try: - fmp_symbols = await _fetch_universe_symbols_from_fmp(normalised_universe) - cleaned_fmp = _normalise_symbols(fmp_symbols) - if cleaned_fmp: - await _write_cached_symbols(db, normalised_universe, cleaned_fmp, "fmp") - return cleaned_fmp, "fmp" - except (ProviderError, ValidationError) as exc: - failures.append(str(exc)) - cached_symbols = await _read_cached_symbols(db, normalised_universe) if cached_symbols: logger.warning( diff --git a/deploy/provision_fundamentals.sh b/deploy/provision_fundamentals.sh index f80d6c6..1590b53 100755 --- a/deploy/provision_fundamentals.sh +++ b/deploy/provision_fundamentals.sh @@ -15,7 +15,6 @@ MIN_FREE_GB="${DOLT_MIN_FREE_DISK_GB:-5}" EARNINGS_DIR="${DOLT_DATA_DIR}/${DOLT_EARNINGS_SUBDIR}" DOLT_IDENTITY_NAME="${DOLT_IDENTITY_NAME:-Signal Platform}" DOLT_IDENTITY_EMAIL="${DOLT_IDENTITY_EMAIL:-signal-platform@localhost}" -FUNDAMENTALS_PARITY_REPORT_DIR="${FUNDAMENTALS_PARITY_REPORT_DIR:-/var/lib/signal-platform/reports/fundamentals-parity}" fail() { echo "ERROR: $*" >&2 @@ -80,8 +79,6 @@ check_env() { || fail "set DOLT_EARNINGS_SUBDIR=$DOLT_EARNINGS_SUBDIR in $ENV_FILE" grep -Eq '^SEC_USER_AGENT=.*@.*' "$ENV_FILE" \ || fail "SEC_USER_AGENT in $ENV_FILE must contain a real contact email" - grep -Fqx "FUNDAMENTALS_PARITY_REPORT_DIR=$FUNDAMENTALS_PARITY_REPORT_DIR" "$ENV_FILE" \ - || fail "set FUNDAMENTALS_PARITY_REPORT_DIR=$FUNDAMENTALS_PARITY_REPORT_DIR in $ENV_FILE" } check_all() { @@ -104,15 +101,6 @@ check_all() { identity_email="$(repo_config_value user.email 2>/dev/null || true)" [[ -n "$identity_name" ]] || fail "missing Dolt user.name for $EARNINGS_DIR" [[ -n "$identity_email" ]] || fail "missing Dolt user.email for $EARNINGS_DIR" - [[ -d "$FUNDAMENTALS_PARITY_REPORT_DIR" ]] \ - || fail "missing parity report directory: $FUNDAMENTALS_PARITY_REPORT_DIR" - if [[ "$(id -un)" == "$APP_USER" ]]; then - [[ -w "$FUNDAMENTALS_PARITY_REPORT_DIR" ]] \ - || fail "parity report directory is not writable by $APP_USER" - else - runuser -u "$APP_USER" -- test -w "$FUNDAMENTALS_PARITY_REPORT_DIR" \ - || fail "parity report directory is not writable by $APP_USER" - fi check_free_space check_env echo "OK: Dolt $DOLT_VERSION and earnings clone are provisioned" @@ -139,7 +127,6 @@ fi version_ok || fail "Dolt $DOLT_VERSION installation failed" install -d -o "$APP_USER" -g "$APP_GROUP" -m 0750 "$DOLT_DATA_DIR" -install -d -o "$APP_USER" -g "$APP_GROUP" -m 0750 "$FUNDAMENTALS_PARITY_REPORT_DIR" check_free_space if [[ ! -d "$EARNINGS_DIR/.dolt" ]]; then diff --git a/docs/dolt-integration-plan.md b/docs/dolt-integration-plan.md index 6c58781..60180af 100644 --- a/docs/dolt-integration-plan.md +++ b/docs/dolt-integration-plan.md @@ -430,7 +430,11 @@ workstream B — Alpaca remains the price source throughout. approval** — see the handoff section below. Step (c) is implemented behind the default-off `fundamental_data_sec_dolt_cutover_enabled` SystemSetting; the remaining production action is flipping that switch on and observing it. -- A6. Remove FMP/Finnhub/Alpha Vantage; keep monitoring + manual fallback. +- A6. **DONE 2026-08-07.** FMP/Finnhub/Alpha Vantage removed, along with the + weekly `fundamental_collector` job, the A5 cutover toggle (SEC+Dolt is now the + unconditional path) and the parity report. Migration `029` tombstones the two + behavior-bearing settings rows for the rollback window; the archived parity + bundles stay as the A5 evidence trail. **Workstream B (independent, start when wanted):** @@ -507,11 +511,13 @@ managed by the **Fundamentals data source** card in Admin → Settings; while ac the weekly legacy collector skips itself so it cannot overwrite the SEC/Dolt cache. See `docs/fundamentals-deployment.md` for the production flip and rollback procedure. -**Task 2 — A6 decommissioning.** After a short observation window: remove -FMP/Finnhub/Alpha Vantage providers, config and env keys; keep monitoring + manual -fallback. Gated by the acceptance criteria above — especially forward-calendar -timeliness from `dolt_earnings` (its `source_max_date` ran ~5 weeks ahead as of -2026-07-23, which passes). +**Task 2 — A6 decommissioning: DONE 2026-08-07.** The cutover ran on and was +observed in production, so the legacy providers, their config/env keys, the weekly +collector job and the parity report were all removed. Two consequences to carry: +(1) `fundamental_data` now has no provider fallback — recovery is restore-from-backup; +(2) disabling **SEC Fundamentals Import** stops the SEC fetch only, because the local +cache refresh was deliberately moved outside the job-enable check. Remaining +follow-up: delete the migration-029 tombstone rows once the rollback window closes. **Known caveats to carry (documented in the findings report, not bugs to fix):** - KLAC-class post-filing splits: P/E wrong until the next 10-Q; undetectable from diff --git a/docs/fundamentals-deployment.md b/docs/fundamentals-deployment.md index a648a33..06df284 100644 --- a/docs/fundamentals-deployment.md +++ b/docs/fundamentals-deployment.md @@ -1,17 +1,21 @@ # Fundamentals production deployment This is the one-time production setup for the Dolt earnings and SEC fundamentals -imports. The A5 scoring cutover was approved on 2026-07-24; the compat-cache write -path is still default-off until the explicit production switch below is set. Do -not add OS cron entries: the application scheduler owns both jobs. +imports. Since A6 (2026-08) these are the *only* fundamentals sources — the +FMP/Finnhub/Alpha Vantage providers, the weekly legacy collector and the A5 parity +report are gone, and the cache write path is unconditional. Do not add OS cron +entries: the application scheduler owns both jobs. ## What the deployment adds -- `Dolt Earnings Import (shadow)` runs daily at 02:30 America/New_York. -- `SEC Fundamentals Import` runs daily at 04:00 America/New_York. Its local - `fundamental_data` refresh runs only when the A5 switch is enabled. -- `Fundamentals Parity Report (read-only)` runs daily at 05:30 America/New_York. +- `Dolt Earnings Import` runs daily at 02:30 America/New_York. +- `SEC Fundamentals Import` runs daily at 04:00 America/New_York, then refreshes + `fundamental_data` — the compat cache scoring reads — from stored snapshots, + earnings events and closes. - Both jobs are visible, toggleable, and manually triggerable in Admin → Jobs. + **Disabling the SEC job stops its SEC network fetch only**; the local cache + refresh still runs, because prices and earnings move daily even when no filing + does. - Cron expressions are editable in Admin → Schedule. - Every attempt is recorded in `data_import_runs`; failures also create a system event. A failed validation does not promote partial data. @@ -36,14 +40,16 @@ DOLT_EARNINGS_SUBDIR=earnings DOLT_MIN_FREE_DISK_GB=5.0 SEC_USER_AGENT=signal-platform/1.0 (contact: real-address@example.com) SEC_REQUEST_SPACING_SECONDS=0.2 -FUNDAMENTALS_PARITY_REPORT_DIR=/var/lib/signal-platform/reports/fundamentals-parity ``` Use a real monitored contact address. Keep at least 5 GB free at the Dolt data path; 8–10 GB gives comfortable growth headroom. The data directory must stay outside `/opt/signalplatform`, because deployments use `rsync --delete` there. -The parity-report directory is also persistent and owned by the service user; -its small timestamped JSON/CSV bundles form the temporary A5 review trail. + +`FMP_API_KEY`, `FINNHUB_API_KEY` and `ALPHA_VANTAGE_API_KEY` must be **removed** +from this file. Nothing reads them any more, and leaving them installed is the +one thing that would let a rolled-back pre-A6 process resume the legacy +collector and overwrite the SEC/Dolt cache. ## One-time provisioning @@ -91,27 +97,7 @@ In Admin → Jobs, wait until no other job is running, then: data and still handles partial/missing issuers cleanly. A ticker held by the quality gate should show **New setups paused** with the specific SEC reason. -## A5 parity observation window - -After both shadow imports are healthy, trigger **Fundamentals Parity Report -(read-only)** once in Admin → Jobs. The **A5 Fundamentals Parity** card above -the jobs shows the latest coverage/delta summary and provides authenticated JSON -and CSV downloads. The canonical server-side bundles are archived at: - -```text -/var/lib/signal-platform/reports/fundamentals-parity/ -``` - -The scheduler then generates one report daily at 05:30 New York time, after the -02:30 Dolt and 04:00 SEC jobs. Review 5–7 consecutive reports before making the -cutover decision. A report never writes `fundamental_data`, dimension/composite -scores, rankings, qualification state, or an approval flag. Materiality bands -only highlight rows for review; A5 still requires explicit approval. - -Each bundle contains legacy and candidate P/E, revenue growth, and earnings -surprise; definition notes; source revisions and price dates; recomputed legacy -and candidate fundamental scores; and per-universe fundamental-rank changes. -Definition changes remain explicit even when numeric deltas are small. +## Verification Optional database verification: @@ -162,45 +148,24 @@ Expect `OK: source lock is busy`. This is the remaining live-PostgreSQL mutual-exclusion check; SQLite unit tests cannot exercise PostgreSQL advisory locks. A second Admin trigger should independently report the job as busy. -## A5 production activation (approved 2026-07-24) +## The fundamentals cache -The write path is controlled by the SystemSetting -`fundamental_data_sec_dolt_cutover_enabled`. An absent value, `false`, or any -value other than `true` leaves `fundamental_data` untouched. Before enabling it, -confirm the normal PostgreSQL backup containing `fundamental_data` is current. +`fundamental_data` is the compat cache scoring reads. The SEC Fundamentals +Import rebuilds it every run from data already in PostgreSQL: newest valid +snapshots x latest close for `pe_ratio` and `market_cap`, snapshots alone for +`revenue_growth`, and `earnings_events` for `earnings_surprise` and +`next_earnings_date`. It therefore also runs after an SEC network/validation +failure, a `no_op`, a source-lock skip, or with the job disabled — no network +access is involved. The job message appends the cache row count and the changed +score-input count. -In **Admin → Settings → Fundamentals data source**: +A refresh marks affected fundamental and composite score caches stale. The +normal 15:30 near-close scanner recomputes them before using the rankings; until +then, reads truthfully expose the stale state. -1. Turn on **Use SEC + Dolt for scoring inputs** and accept the confirmation. -2. Click **Run refresh now**. The SEC import may be `promoted` or `no_op`; either - result runs the local cache refresh. - -The weekly legacy collector is automatically skipped while the switch is on, so -it cannot overwrite the activated cache. The switch remains visible even before -its SystemSetting row exists because the safe default is off. - -If the Admin UI is unavailable, enable the cutover directly in PostgreSQL: +Verify the refreshed rows: ```sql -INSERT INTO system_settings (key, value, updated_at) -VALUES ('fundamental_data_sec_dolt_cutover_enabled', 'true', now()) -ON CONFLICT (key) DO UPDATE -SET value = EXCLUDED.value, updated_at = now(); -``` - -Then trigger **SEC Fundamentals Import** once in Admin → Jobs. Once enabled, the -same refresh also runs after an SEC network/validation failure or a source-lock -skip, because it reads only PostgreSQL snapshots, earnings events, and closes. -The job message appends the cache row count and changed score-input count when -the import itself completed successfully. - -Verify the switch and refreshed rows: - -```sql -SELECT key, value, updated_at -FROM system_settings -WHERE key = 'fundamental_data_sec_dolt_cutover_enabled'; - SELECT count(*) AS rows, max(fetched_at) AS refreshed_at, count(pe_ratio) AS pe_available, @@ -213,28 +178,33 @@ SELECT dimension, is_stale, count(*) FROM dimension_scores WHERE dimension = 'fundamental' GROUP BY dimension, is_stale; - -SELECT is_stale, count(*) -FROM composite_scores -GROUP BY is_stale; ``` -The first refresh intentionally marks affected fundamental and composite score -caches stale. The normal 15:30 near-close scanner recomputes them before using -the rankings; until then, reads truthfully expose the stale state. Observe at -least several scheduled cycles before A6 removes the legacy providers. +### Retired settings rows (delete later) + +Migration `029` pinned two SystemSetting rows as rollback tombstones rather than +deleting them, because migrations run before the service restarts and a +rolled-back pre-A6 process reads an absent `job_..._enabled` row as *enabled*: + +| key | pinned value | +|---|---| +| `fundamental_data_sec_dolt_cutover_enabled` | `true` | +| `job_fundamental_collector_enabled` | `false` | + +Neither controls anything now; Admin -> Settings hides both. Once the A6 +rollback window has closed, delete the rows and the `MANAGED_SETTINGS` filter in +`frontend/src/components/admin/SettingsForm.tsx`. ## Failure and rollback -- To stop the A5 cache writes without stopping SEC snapshot ingestion, turn off - **Use SEC + Dolt for scoring inputs** in Admin → Settings. If the UI is - unavailable, set `fundamental_data_sec_dolt_cutover_enabled` back to `false` - with the SQL above (changing only the value). This prevents the next local - refresh but does not restore rows already replaced. Restore `fundamental_data` - from the pre-cutover database backup, or—before A6—manually run the legacy - Fundamental Collector if its provider keys and quota are still available. -- Disable a failing source-import job in Admin → Jobs only when ingestion itself - must stop. Existing promoted snapshots/events remain available. +- **There is no provider fallback any more.** To recover bad `fundamental_data` + values, restore the table from the PostgreSQL backup; the next scheduled run + will rebuild it from the current snapshots. Confirm a recent backup exists + before any change that could corrupt the snapshots. +- Disable a failing source-import job in Admin → Jobs only when SEC network + access itself must stop — the local cache refresh keeps running, and existing + promoted snapshots/events remain available. To freeze the cache as well, + disable the job *and* accept that P/E, market cap and earnings dates go stale. - Inspect the job runtime, latest `data_import_runs.validation_json`, service logs, and Admin → System Events before retrying. - `unresolved_filing` is emitted once when a filing enters automatic retry. It @@ -250,8 +220,3 @@ least several scheduled cycles before A6 removes the legacy providers. - The Dolt clone is a reproducible cache and does not need a bespoke backup. PostgreSQL (including `earnings_events`, `fundamental_snapshots`, and import audit rows) must remain covered by the normal production database backup. -- Do not proceed to A6 until the activated cache has completed the observation - window and the forward earnings calendar remains timely. -- If report generation fails, inspect Admin → System Events and verify - `FUNDAMENTALS_PARITY_REPORT_DIR` exists and is writable by `deploy`. Existing - reports and all live data remain untouched. diff --git a/frontend/src/api/admin.ts b/frontend/src/api/admin.ts index d24ff9e..20993a4 100644 --- a/frontend/src/api/admin.ts +++ b/frontend/src/api/admin.ts @@ -4,7 +4,6 @@ import type { AdminUser, AlertConfig, AlertTestResult, - FundamentalsCutoverConfig, PipelineReadiness, RecommendationConfig, ScheduleConfig, @@ -57,18 +56,6 @@ export function updateSetting(key: string, value: string) { .then((r) => r.data); } -export function getFundamentalsCutoverSettings() { - return apiClient - .get('admin/settings/fundamentals-cutover') - .then((r) => r.data); -} - -export function updateFundamentalsCutoverSettings(enabled: boolean) { - return apiClient - .put('admin/settings/fundamentals-cutover', { enabled }) - .then((r) => r.data); -} - export function getRecommendationSettings() { return apiClient .get('admin/settings/recommendations') @@ -246,40 +233,6 @@ export interface TriggerJobResponse { cadence?: BacktestCadence; } -export interface ParityFieldStats { - legacy_available: number; - candidate_available: number; - both_available: number; - material_differences: number; - median_absolute_delta: number | null; - p95_absolute_delta: number | null; - max_absolute_delta: number | null; -} - -export interface FundamentalsParityReport { - report_version: number; - generated_at: string; - as_of_date: string; - approval_status: string; - read_only: boolean; - summary: { - universe_count: number; - legacy_fundamental_score_available: number; - candidate_fundamental_score_available: number; - fundamental_scores_compared: number; - fundamental_score_material_changes: number; - fundamental_rank_changes: number; - field_stats: Record; - }; - source_runs: Record; -} - export type BacktestTargetModel = 'production_gtl' | 'structural_sr'; export type BacktestCadence = 'weekly' | 'daily'; @@ -306,24 +259,6 @@ export function triggerJob( .then((r) => r.data); } -export function getFundamentalsParityReport() { - return apiClient - .get('admin/fundamentals-parity') - .then((r) => r.data); -} - -export function getFundamentalsParityCsv() { - return apiClient - .get<{ filename: string; content: string } | null>('admin/fundamentals-parity/csv') - .then((r) => r.data); -} - -export function getFundamentalsParityJson() { - return apiClient - .get<{ filename: string; content: string } | null>('admin/fundamentals-parity/json') - .then((r) => r.data); -} - // System events (operational warnings / errors) export interface SystemEvent { id: number; diff --git a/frontend/src/api/ingestion.ts b/frontend/src/api/ingestion.ts index 02fbb1c..e81f645 100644 --- a/frontend/src/api/ingestion.ts +++ b/frontend/src/api/ingestion.ts @@ -14,7 +14,7 @@ export interface FetchDataResult { } /** Provider sources that cost an API call/quota. */ -export type FetchSource = 'ohlcv' | 'sentiment' | 'fundamentals'; +export type FetchSource = 'ohlcv' | 'sentiment'; /** Source selector: omit → fetch all; array → those providers; 'recompute' → derived only (free). */ export type FetchSelector = FetchSource[] | 'recompute'; diff --git a/frontend/src/components/admin/FundamentalsCutoverSettings.tsx b/frontend/src/components/admin/FundamentalsCutoverSettings.tsx deleted file mode 100644 index 4c58c41..0000000 --- a/frontend/src/components/admin/FundamentalsCutoverSettings.tsx +++ /dev/null @@ -1,176 +0,0 @@ -import { - useFundamentalsCutoverSettings, - useJobs, - useTriggerJob, - useUpdateFundamentalsCutoverSettings, -} from '../../hooks/useAdmin'; -import { SkeletonCard } from '../ui/Skeleton'; - -const SEC_JOB = 'sec_fundamentals_import'; - -function formatRun(iso: string | null | undefined): string { - if (!iso) return 'not run in this process'; - const minutes = Math.floor((Date.now() - new Date(iso).getTime()) / 60_000); - if (minutes < 1) return 'just now'; - if (minutes < 60) return `${minutes}m ago`; - const hours = Math.floor(minutes / 60); - return hours < 24 ? `${hours}h ago` : `${Math.floor(hours / 24)}d ago`; -} - -export function FundamentalsCutoverSettings() { - const cutover = useFundamentalsCutoverSettings(); - const update = useUpdateFundamentalsCutoverSettings(); - const trigger = useTriggerJob(); - const { data: jobs } = useJobs(); - - if (cutover.isLoading) return ; - if (cutover.isError || !cutover.data) { - return ( -

- {(cutover.error as Error)?.message || 'Failed to load fundamentals data source'} -

- ); - } - - const enabled = cutover.data.enabled; - const secJob = jobs?.find((job) => job.name === SEC_JOB); - const runningJob = jobs?.find((job) => job.running); - const refreshBlocked = Boolean(runningJob && runningJob.name !== SEC_JOB); - - const changeSource = () => { - const next = !enabled; - const confirmed = window.confirm( - next - ? 'Activate SEC + Dolt fundamentals? The next SEC import will replace the legacy cache and mark affected scores stale.' - : 'Pause SEC + Dolt cache refreshes? Existing cache values will stay in place; legacy values are not restored automatically.', - ); - if (confirmed) update.mutate(next); - }; - - return ( -
-
-
-
-
-
-

- Fundamentals data source -

- - {enabled ? 'SEC + Dolt active' : 'Legacy cache'} - -
-

- Controls what repopulates fundamental_data, the - compatibility cache used by scoring. SEC filings supply P/E, growth and estimated market - cap; Dolt supplies earnings dates and surprises. Everything is derived locally from PostgreSQL. -

-
-
- -
-
-
Legacy APIs
-
FMP / Finnhub / Alpha Vantage
-
- -
-
SEC + Dolt
-
Bulk imports → PostgreSQL cache
-
-
- -
-
-
-
1 · Source
-
Use SEC + Dolt for scoring inputs
-

- While active, the weekly legacy collector is skipped so it cannot overwrite the new cache. -

-
- -
- -
-
2 · Refresh
-
-
-
Apply the source now
-

- {secJob?.running - ? 'SEC import and cache refresh are running.' - : secJob?.runtime_message || `Last SEC run: ${formatRun(secJob?.runtime_finished_at)}`} -

-
- -
- {!enabled && ( -

Activate the source before running the refresh.

- )} - {enabled && secJob?.enabled === false && ( -

Enable the SEC Fundamentals job on the Jobs tab first.

- )} -
-
- -

- Rollback pauses future writes only. To restore pre-cutover values, use the database backup or - pause this source and manually run the legacy collector while its provider keys remain installed. -

-
-
- ); -} diff --git a/frontend/src/components/admin/FundamentalsParityPanel.tsx b/frontend/src/components/admin/FundamentalsParityPanel.tsx deleted file mode 100644 index 36be387..0000000 --- a/frontend/src/components/admin/FundamentalsParityPanel.tsx +++ /dev/null @@ -1,156 +0,0 @@ -import { useState } from 'react'; -import { - getFundamentalsParityCsv, - getFundamentalsParityJson, -} from '../../api/admin'; -import { useFundamentalsParityReport } from '../../hooks/useAdmin'; -import { SkeletonTable } from '../ui/Skeleton'; - -const FIELD_LABELS: Record = { - pe_ratio: 'P/E', - revenue_growth: 'Revenue growth', - earnings_surprise: 'Earnings surprise', -}; - -function downloadText(filename: string, content: string, type: string) { - const blob = new Blob([content], { type }); - const url = URL.createObjectURL(blob); - const anchor = document.createElement('a'); - anchor.href = url; - anchor.download = filename; - anchor.click(); - URL.revokeObjectURL(url); -} - -export function FundamentalsParityPanel() { - const { data: report, isLoading, isError, error } = useFundamentalsParityReport(); - const [downloading, setDownloading] = useState(false); - - if (isLoading) return ; - if (isError) { - return

{(error as Error).message}

; - } - - if (!report) { - return ( -
-

A5 Fundamentals Parity

-

- No report yet. Trigger “Fundamentals Parity Report (read-only)” below. -

-
- ); - } - - const summary = report.summary; - const generated = new Date(report.generated_at).toLocaleString(); - - async function downloadCsv() { - setDownloading(true); - try { - const artifact = await getFundamentalsParityCsv(); - if (artifact) downloadText(artifact.filename, artifact.content, 'text/csv;charset=utf-8'); - } finally { - setDownloading(false); - } - } - - async function downloadJson() { - setDownloading(true); - try { - const artifact = await getFundamentalsParityJson(); - if (artifact) downloadText(artifact.filename, artifact.content, 'application/json;charset=utf-8'); - } finally { - setDownloading(false); - } - } - - return ( -
-
-
-
-

A5 Fundamentals Parity

- - approval pending - - - read-only - -
-

- Generated {generated} · as of {report.as_of_date} · {summary.universe_count} tracked tickers -

-
-
- - -
-
- -
- - - - -
- -
- - - - - - - - - - - - - {Object.entries(summary.field_stats).map(([key, stats]) => ( - - - - - - - - - ))} - -
FieldLegacyCandidateComparedMaterialMedian |Δ|
{FIELD_LABELS[key] ?? key}{stats.legacy_available}{stats.candidate_available}{stats.both_available}{stats.material_differences} - {stats.median_absolute_delta == null ? 'n/a' : stats.median_absolute_delta.toFixed(2)} -
-
- -

- Materiality bands highlight review candidates only. They do not approve a cutover or write fundamentals, - scores, rankings, or qualification state. -

-
- ); -} - -function Summary({ label, value }: { label: string; value: string | number }) { - return ( -
-
{label}
-
{value}
-
- ); -} diff --git a/frontend/src/components/admin/ScheduleSettings.tsx b/frontend/src/components/admin/ScheduleSettings.tsx index e084fc6..4d2f21c 100644 --- a/frontend/src/components/admin/ScheduleSettings.tsx +++ b/frontend/src/components/admin/ScheduleSettings.tsx @@ -8,11 +8,9 @@ const DEFAULTS: ScheduleConfig = { schedule_daily_pipeline_cron: '0 2 * * *', schedule_dolt_earnings_cron: '30 2 * * *', schedule_sec_fundamentals_cron: '0 4 * * *', - schedule_fundamentals_parity_cron: '30 5 * * *', schedule_near_close_pipeline_cron: '30 15 * * mon-fri', schedule_after_close_pipeline_cron: '45 16 * * mon-fri', schedule_intraday_pipeline_cron: '0 10-15 * * mon-fri', - schedule_fundamentals_cron: '0 1 * * mon', }; const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: boolean }[] = [ @@ -30,19 +28,13 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b { key: 'schedule_dolt_earnings_cron', label: 'Dolt earnings', - hint: 'Pull and import earnings dates/results daily at 02:30 ET. The activated cache refresh uses these local events.', + hint: 'Pull and import earnings dates/results daily at 02:30 ET. The fundamentals cache refresh uses these local events.', mono: true, }, { key: 'schedule_sec_fundamentals_cron', label: 'SEC fundamentals', - hint: 'Import tracked-universe SEC facts daily at 04:00 ET and refresh the scoring cache when the cutover is active.', - mono: true, - }, - { - key: 'schedule_fundamentals_parity_cron', - label: 'Fundamentals parity report', - hint: 'Read-only legacy vs SEC/Dolt comparison daily at 05:30 ET, after the bulk imports.', + hint: 'Import tracked-universe SEC facts daily at 04:00 ET, then refresh the fundamentals cache scoring reads. Disabling the job stops the SEC fetch only — the local cache refresh still runs.', mono: true, }, { @@ -63,12 +55,6 @@ const FIELDS: { key: keyof ScheduleConfig; label: string; hint: string; mono?: b hint: 'Refresh prices + resolve outcomes mid-session. Default hourly 10:00–15:00 ET weekdays.', mono: true, }, - { - key: 'schedule_fundamentals_cron', - label: 'Legacy fundamentals (weekly)', - hint: 'Fallback provider chain. Automatically skipped while the SEC + Dolt cutover is active.', - mono: true, - }, ]; export function ScheduleSettings() { diff --git a/frontend/src/components/admin/SettingsForm.tsx b/frontend/src/components/admin/SettingsForm.tsx index cdd2993..a4d7165 100644 --- a/frontend/src/components/admin/SettingsForm.tsx +++ b/frontend/src/components/admin/SettingsForm.tsx @@ -3,7 +3,13 @@ import { useSettings, useUpdateSetting } from '../../hooks/useAdmin'; import { SkeletonTable } from '../ui/Skeleton'; import type { SystemSetting } from '../../lib/types'; -const MANAGED_SETTINGS = new Set(['fundamental_data_sec_dolt_cutover_enabled']); +// Retired keys kept in the database as rollback tombstones (migration 029). +// They no longer control anything; hide them so nobody edits a dead switch. +// Delete both the rows and this filter once the A6 rollback window has closed. +const MANAGED_SETTINGS = new Set([ + 'fundamental_data_sec_dolt_cutover_enabled', + 'job_fundamental_collector_enabled', +]); export function SettingsForm() { const { data: settings, isLoading, isError, error } = useSettings(); diff --git a/frontend/src/hooks/useAdmin.ts b/frontend/src/hooks/useAdmin.ts index f0da952..7d2d6b2 100644 --- a/frontend/src/hooks/useAdmin.ts +++ b/frontend/src/hooks/useAdmin.ts @@ -90,36 +90,6 @@ export function useUpdateSetting() { }); } -export function useFundamentalsCutoverSettings() { - return useQuery({ - queryKey: ['admin', 'fundamentals-cutover'], - queryFn: () => adminApi.getFundamentalsCutoverSettings(), - }); -} - -export function useUpdateFundamentalsCutoverSettings() { - const qc = useQueryClient(); - const { addToast } = useToast(); - - return useMutation({ - mutationFn: (enabled: boolean) => - adminApi.updateFundamentalsCutoverSettings(enabled), - onSuccess: (config) => { - qc.setQueryData(['admin', 'fundamentals-cutover'], config); - qc.invalidateQueries({ queryKey: ['admin', 'settings'] }); - addToast( - config.enabled ? 'success' : 'info', - config.enabled - ? 'SEC + Dolt fundamentals activated' - : 'SEC + Dolt cache refresh paused', - ); - }, - onError: (error: Error) => { - addToast('error', error.message || 'Failed to update fundamentals data source'); - }, - }); -} - export function useRecommendationSettings() { return useQuery({ queryKey: ['admin', 'recommendation-settings'], @@ -346,14 +316,6 @@ export function useJobs() { }); } -export function useFundamentalsParityReport() { - return useQuery({ - queryKey: ['admin', 'fundamentals-parity'], - queryFn: () => adminApi.getFundamentalsParityReport(), - refetchInterval: 15_000, - }); -} - export function usePipelineReadiness() { return useQuery({ queryKey: ['admin', 'pipeline-readiness'], diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index 991a658..f5ee7b2 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -187,21 +187,15 @@ export interface ActivationConfig { exclude_neutral: boolean; } -export interface FundamentalsCutoverConfig { - enabled: boolean; -} - // Cron schedule for morning / near-close / after-close / intraday + fundamentals export interface ScheduleConfig { schedule_timezone: string; schedule_daily_pipeline_cron: string; schedule_dolt_earnings_cron: string; schedule_sec_fundamentals_cron: string; - schedule_fundamentals_parity_cron: string; schedule_near_close_pipeline_cron: string; schedule_after_close_pipeline_cron: string; schedule_intraday_pipeline_cron: string; - schedule_fundamentals_cron: string; } // Runtime sentiment LLM configuration @@ -892,7 +886,7 @@ export interface TickerUniverseSetting { export interface TickerUniverseBootstrapResult { universe: TickerUniverse; - /** Where the member list came from: wikipedia_sp500 | fmp | cache | seed | … */ + /** Where the member list came from: wikipedia_sp500 | nasdaq_trader | cache | seed | … */ source?: string; total_universe_symbols: number; added: number; diff --git a/frontend/src/pages/AdminPage.tsx b/frontend/src/pages/AdminPage.tsx index 6ec3d47..57f11e2 100644 --- a/frontend/src/pages/AdminPage.tsx +++ b/frontend/src/pages/AdminPage.tsx @@ -5,8 +5,6 @@ import { AlertSettings } from '../components/admin/AlertSettings'; import { SentimentProviderSettings } from '../components/admin/SentimentProviderSettings'; import { DataCleanup } from '../components/admin/DataCleanup'; import { JobControls } from '../components/admin/JobControls'; -import { FundamentalsParityPanel } from '../components/admin/FundamentalsParityPanel'; -import { FundamentalsCutoverSettings } from '../components/admin/FundamentalsCutoverSettings'; import { PerformanceSettings } from '../components/admin/PerformanceSettings'; import { PipelineReadinessPanel } from '../components/admin/PipelineReadinessPanel'; import { SystemEventsPanel } from '../components/admin/SystemEventsPanel'; @@ -37,7 +35,6 @@ export default function AdminPage() { {activeTab === 'Tickers' && } {activeTab === 'Settings' && (
- @@ -51,7 +48,6 @@ export default function AdminPage() { {activeTab === 'Jobs' && (
-
diff --git a/frontend/src/pages/TickerDetailPage.tsx b/frontend/src/pages/TickerDetailPage.tsx index d6900c2..3804e8c 100644 --- a/frontend/src/pages/TickerDetailPage.tsx +++ b/frontend/src/pages/TickerDetailPage.tsx @@ -102,7 +102,7 @@ interface DataStatusItem { available: boolean; timestamp?: string | null; timestampLabel?: string | null; - selector: FetchSelector; // what a refresh of this row fetches + selector?: FetchSelector; // what a refresh fetches; omit for rows with no manual refresh paid?: boolean; // provider call that may cost money/quota } @@ -138,14 +138,16 @@ function DataFreshnessBar({ ) : !item.available ? ( no data ) : null} - + {item.selector && ( + + )} {item.paid && $}
))} @@ -226,11 +228,11 @@ export default function TickerDetailPage() { paid: true, }, { + // Rebuilt for the whole universe by the nightly SEC + Dolt imports — + // there is no per-ticker fetch to offer here. label: 'Fundamentals', available: !!fundamentals.data && fundamentals.data.fetched_at !== null, timestamp: fundamentals.data?.fetched_at, - selector: ['fundamentals'] as FetchSelector, - paid: true, }, { label: 'S/R Levels', @@ -247,6 +249,7 @@ export default function TickerDetailPage() { ], [ohlcv.data, sentiment.data, fundamentals.data, srLevels.data, scores.data]); const handleRefresh = (item: DataStatusItem) => { + if (!item.selector) return; setRefreshingLabel(item.label); ingestion.mutate( { symbol, sources: item.selector }, diff --git a/scripts/backfill_earnings_events.py b/scripts/backfill_earnings_events.py deleted file mode 100644 index 570e117..0000000 --- a/scripts/backfill_earnings_events.py +++ /dev/null @@ -1,515 +0,0 @@ -"""Bulk-only historical earnings backfill for a local SQLite snapshot. - -The job uses FMP's date-range earnings-calendar endpoint. One request covers all -symbols in a date window; per-symbol endpoints are intentionally not available -in this task runner. Successful windows are committed independently so a later -run resumes after a daily quota boundary without repeating completed windows. - -Example: - python scripts/backfill_earnings_events.py --snapshot backtest_snapshots/prod.sqlite \ - --from-date 2012-01-01 --window-days 30 --limit 250 -""" - -from __future__ import annotations - -import argparse -import asyncio -import json -import math -import sys -from datetime import date, datetime, timedelta, timezone -from pathlib import Path -from typing import Any - -import httpx -from sqlalchemy import create_engine, text - -ROOT = Path(__file__).resolve().parents[1] -if str(ROOT) not in sys.path: - sys.path.insert(0, str(ROOT)) - -from app.ssl_bootstrap import bootstrap_ssl # noqa: E402 - -bootstrap_ssl() - -FMP_STABLE = "https://financialmodelingprep.com/stable" -EVENTS_DDL = """ -CREATE TABLE IF NOT EXISTS earnings_events ( - id INTEGER PRIMARY KEY, - symbol TEXT NOT NULL, - announce_date TEXT NOT NULL, - announce_time TEXT, - eps_estimate REAL, - eps_actual REAL, - revenue_estimate REAL, - revenue_actual REAL, - source TEXT NOT NULL, - fetched_at TEXT NOT NULL, - UNIQUE(symbol, announce_date) -) -""" -META_DDL = """ -CREATE TABLE IF NOT EXISTS earnings_backfill_meta ( - symbol TEXT PRIMARY KEY, - status TEXT NOT NULL, - n_events INTEGER NOT NULL DEFAULT 0, - updated_at TEXT NOT NULL, - note TEXT -) -""" -WINDOW_DDL = """ -CREATE TABLE IF NOT EXISTS earnings_backfill_windows ( - from_date TEXT NOT NULL, - to_date TEXT NOT NULL, - status TEXT NOT NULL, - requests INTEGER NOT NULL DEFAULT 0, - rows_raw INTEGER NOT NULL DEFAULT 0, - rows_universe INTEGER NOT NULL DEFAULT 0, - duplicate_rows INTEGER NOT NULL DEFAULT 0, - restated_rows INTEGER NOT NULL DEFAULT 0, - updated_at TEXT NOT NULL, - note TEXT, - PRIMARY KEY(from_date, to_date) -) -""" - - -def _parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--snapshot", default="backtest_snapshots/prod.sqlite") - parser.add_argument("--from-date", default="2012-01-01") - parser.add_argument("--to-date", default=None) - parser.add_argument("--window-days", type=int, default=30) - parser.add_argument("--limit", type=int, default=250) - parser.add_argument("--sleep", type=float, default=0.35) - parser.add_argument( - "--refetch-windows", - action="store_true", - help="Re-fetch date windows already logged as done.", - ) - return parser.parse_args() - - -def _ensure_tables(engine) -> None: - with engine.begin() as conn: - conn.execute(text(EVENTS_DDL)) - conn.execute(text(META_DDL)) - conn.execute(text(WINDOW_DDL)) - - -def _number(value: Any) -> float | None: - if value is None or value == "": - return None - try: - result = float(value) - except (TypeError, ValueError): - return None - return result if math.isfinite(result) else None - - -def _normalise_session(value: Any) -> str | None: - if value is None: - return None - cleaned = str(value).strip().lower().replace("_", " ").replace("-", " ") - aliases = { - "bmo": "bmo", - "before market open": "bmo", - "before open": "bmo", - "amc": "amc", - "after market close": "amc", - "after close": "amc", - "during market hours": "during", - "dmh": "during", - } - return aliases.get(cleaned, cleaned or None) - - -def _parse_bulk_item(item: dict) -> dict | None: - symbol = str(item.get("symbol") or "").strip().upper().replace(".", "-") - raw_date = item.get("date") or item.get("earningsDate") - if not symbol or not raw_date: - return None - return { - "symbol": symbol, - "announce_date": str(raw_date)[:10], - "announce_time": _normalise_session( - item.get("time") or item.get("announceTime") - ), - "eps_estimate": _number( - item.get("epsEstimated") - if item.get("epsEstimated") is not None - else item.get("estimatedEarning") - ), - "eps_actual": _number( - item.get("epsActual") - if item.get("epsActual") is not None - else item.get("eps") - ), - "revenue_estimate": _number(item.get("revenueEstimated")), - "revenue_actual": _number(item.get("revenueActual")), - } - - -def _windows(start: date, end: date, window_days: int) -> list[tuple[date, date]]: - if window_days < 1: - raise ValueError("window_days must be positive") - result: list[tuple[date, date]] = [] - cursor = start - while cursor <= end: - window_end = min(end, cursor + timedelta(days=window_days - 1)) - result.append((cursor, window_end)) - cursor = window_end + timedelta(days=1) - return result - - -def _dedupe_bulk_rows(rows: list[dict]) -> tuple[list[dict], int, int]: - """Prefer the most complete duplicate; use the later row as the tie-break.""" - fields = ( - "announce_time", - "eps_estimate", - "eps_actual", - "revenue_estimate", - "revenue_actual", - ) - chosen: dict[tuple[str, str], dict] = {} - duplicate_extras = 0 - restated = 0 - for row in rows: - key = (str(row["symbol"]), str(row["announce_date"])) - previous = chosen.get(key) - if previous is None: - chosen[key] = row - continue - duplicate_extras += 1 - if any( - previous.get(field) is not None - and row.get(field) is not None - and previous.get(field) != row.get(field) - for field in fields - ): - restated += 1 - previous_score = sum(previous.get(field) is not None for field in fields) - new_score = sum(row.get(field) is not None for field in fields) - if new_score >= previous_score: - chosen[key] = row - return list(chosen.values()), duplicate_extras, restated - - -def _upsert_events(conn, rows: list[dict]) -> int: - if not rows: - return 0 - fetched_at = datetime.now(timezone.utc).isoformat() - statement = text( - """ - INSERT INTO earnings_events ( - symbol, announce_date, announce_time, eps_estimate, eps_actual, - revenue_estimate, revenue_actual, source, fetched_at - ) VALUES ( - :symbol, :announce_date, :announce_time, :eps_estimate, :eps_actual, - :revenue_estimate, :revenue_actual, 'fmp_earnings_calendar', :fetched_at - ) - ON CONFLICT(symbol, announce_date) DO UPDATE SET - announce_time=COALESCE(excluded.announce_time, earnings_events.announce_time), - eps_estimate=COALESCE(excluded.eps_estimate, earnings_events.eps_estimate), - eps_actual=COALESCE(excluded.eps_actual, earnings_events.eps_actual), - revenue_estimate=COALESCE(excluded.revenue_estimate, earnings_events.revenue_estimate), - revenue_actual=COALESCE(excluded.revenue_actual, earnings_events.revenue_actual), - source=excluded.source, - fetched_at=excluded.fetched_at - """ - ) - conn.execute(statement, [{**row, "fetched_at": fetched_at} for row in rows]) - return len(rows) - - -async def _fetch_bulk_window( - client: httpx.AsyncClient, api_key: str, start: date, end: date -) -> tuple[list[dict], int, str | None]: - response = await client.get( - f"{FMP_STABLE}/earnings-calendar", - params={"from": start.isoformat(), "to": end.isoformat(), "apikey": api_key}, - ) - if response.status_code in (402, 403): - return [], response.status_code, "bulk_endpoint_unavailable" - if response.status_code == 429: - return [], response.status_code, "daily_limit_reached" - response.raise_for_status() - payload = response.json() - if not isinstance(payload, list): - return [], response.status_code, f"unexpected_payload:{type(payload).__name__}" - rows = [] - for item in payload: - if isinstance(item, dict): - parsed = _parse_bulk_item(item) - if parsed: - rows.append(parsed) - return rows, response.status_code, None - - -def _write_window_status( - engine, - *, - start: date, - end: date, - status: str, - raw_n: int = 0, - universe_n: int = 0, - duplicate_n: int = 0, - restated_n: int = 0, - note: str | None = None, -) -> None: - with engine.begin() as conn: - conn.execute( - text( - """ - INSERT INTO earnings_backfill_windows( - from_date, to_date, status, requests, rows_raw, rows_universe, - duplicate_rows, restated_rows, updated_at, note - ) VALUES (:a, :b, :status, 1, :raw, :uni, :dup, :rest, :now, :note) - ON CONFLICT(from_date, to_date) DO UPDATE SET - status=excluded.status, - requests=earnings_backfill_windows.requests + 1, - rows_raw=excluded.rows_raw, - rows_universe=excluded.rows_universe, - duplicate_rows=excluded.duplicate_rows, - restated_rows=excluded.restated_rows, - updated_at=excluded.updated_at, - note=excluded.note - """ - ), - { - "a": start.isoformat(), - "b": end.isoformat(), - "status": status, - "raw": raw_n, - "uni": universe_n, - "dup": duplicate_n, - "rest": restated_n, - "now": datetime.now(timezone.utc).isoformat(), - "note": note, - }, - ) - - -async def _main() -> None: - args = _parse_args() - snapshot = Path(args.snapshot) - if not snapshot.exists(): - raise SystemExit(f"Snapshot not found: {snapshot}") - - from app.config import settings - - if not settings.fmp_api_key: - raise SystemExit("FMP_API_KEY required") - start = date.fromisoformat(args.from_date) - end = date.fromisoformat(args.to_date) if args.to_date else date.today() - if start > end: - raise SystemExit("--from-date must not be after --to-date") - - engine = create_engine(f"sqlite:///{snapshot.resolve().as_posix()}", future=True) - _ensure_tables(engine) - all_windows = _windows(start, end, int(args.window_days)) - with engine.connect() as conn: - symbols = [ - str(row[0]).upper().replace(".", "-") - for row in conn.execute(text("SELECT symbol FROM tickers ORDER BY symbol")) - ] - completed = { - (str(row[0]), str(row[1])) - for row in conn.execute( - text( - "SELECT from_date, to_date FROM earnings_backfill_windows " - "WHERE status='done'" - ) - ) - } - pending = [ - window - for window in all_windows - if args.refetch_windows - or (window[0].isoformat(), window[1].isoformat()) not in completed - ] - universe = set(symbols) - print(f"Snapshot: {snapshot}") - print(f"Universe: {len(symbols)} symbols") - print(f"Window: {start} -> {end}") - print( - f"Bulk windows: {len(all_windows)} total; " - f"{len(all_windows) - len(pending)} done; {len(pending)} pending" - ) - print("Provider: FMP bulk earnings-calendar only") - - requests_this_run = 0 - rows_upserted = 0 - duplicate_rows = 0 - restated_rows = 0 - stop_note: str | None = None - async with httpx.AsyncClient(timeout=60.0) as client: - for index, (window_start, window_end) in enumerate(pending, 1): - if requests_this_run >= int(args.limit): - stop_note = "request_budget_exhausted" - break - try: - raw_rows, status_code, error = await _fetch_bulk_window( - client, settings.fmp_api_key, window_start, window_end - ) - except Exception as exc: - raw_rows, status_code = [], 0 - error = f"request_error:{type(exc).__name__}:{exc}" - requests_this_run += 1 - if error: - _write_window_status( - engine, - start=window_start, - end=window_end, - status="error", - note=f"http={status_code} {error}"[:300], - ) - stop_note = error - print( - f"STOP {window_start}..{window_end}: {error} " - f"(http={status_code}, request={requests_this_run})" - ) - break - - in_universe = [row for row in raw_rows if row["symbol"] in universe] - deduped, duplicate_n, restated_n = _dedupe_bulk_rows(in_universe) - with engine.begin() as conn: - rows_upserted += _upsert_events(conn, deduped) - _write_window_status( - engine, - start=window_start, - end=window_end, - status="done", - raw_n=len(raw_rows), - universe_n=len(deduped), - duplicate_n=duplicate_n, - restated_n=restated_n, - note="bulk", - ) - duplicate_rows += duplicate_n - restated_rows += restated_n - if index == 1 or index % 10 == 0 or index == len(pending): - print( - f"progress windows={index}/{len(pending)} " - f"requests={requests_this_run}/{args.limit} " - f"last={window_start}..{window_end} rows={len(deduped)}" - ) - if args.sleep > 0: - await asyncio.sleep(float(args.sleep)) - - with engine.begin() as conn: - windows_done = int( - conn.execute( - text( - "SELECT COUNT(*) FROM earnings_backfill_windows " - "WHERE status='done' AND from_date >= :a AND to_date <= :b" - ), - {"a": start.isoformat(), "b": end.isoformat()}, - ).scalar_one() - ) - complete = windows_done >= len(all_windows) - if complete: - now = datetime.now(timezone.utc).isoformat() - for symbol in symbols: - count = int( - conn.execute( - text( - "SELECT COUNT(*) FROM earnings_events " - "WHERE symbol=:symbol AND announce_date BETWEEN :a AND :b" - ), - {"symbol": symbol, "a": start.isoformat(), "b": end.isoformat()}, - ).scalar_one() - ) - conn.execute( - text( - """ - INSERT INTO earnings_backfill_meta(symbol, status, n_events, updated_at, note) - VALUES (:symbol, 'done', :count, :now, 'bulk_complete') - ON CONFLICT(symbol) DO UPDATE SET - status='done', n_events=excluded.n_events, - updated_at=excluded.updated_at, note=excluded.note - """ - ), - {"symbol": symbol, "count": count, "now": now}, - ) - params = {"a": start.isoformat(), "b": end.isoformat()} - total_events = int( - conn.execute( - text( - "SELECT COUNT(*) FROM earnings_events " - "WHERE symbol IN (SELECT symbol FROM tickers) " - "AND announce_date BETWEEN :a AND :b" - ), - params, - ).scalar_one() - ) - paired_events = int( - conn.execute( - text( - "SELECT COUNT(*) FROM earnings_events " - "WHERE symbol IN (SELECT symbol FROM tickers) " - "AND announce_date BETWEEN :a AND :b " - "AND eps_actual IS NOT NULL AND eps_estimate IS NOT NULL" - ), - params, - ).scalar_one() - ) - date_range = conn.execute( - text( - "SELECT MIN(announce_date), MAX(announce_date) FROM earnings_events " - "WHERE symbol IN (SELECT symbol FROM tickers) " - "AND announce_date BETWEEN :a AND :b" - ), - params, - ).fetchone() - done_symbols = int( - conn.execute( - text("SELECT COUNT(*) FROM earnings_backfill_meta WHERE status='done'") - ).scalar_one() - ) - totals = conn.execute( - text( - "SELECT COALESCE(SUM(requests),0), COALESCE(SUM(duplicate_rows),0), " - "COALESCE(SUM(restated_rows),0) FROM earnings_backfill_windows " - "WHERE from_date >= :a AND to_date <= :b" - ), - params, - ).fetchone() - - summary = { - "mode": "fmp_bulk_date_range_only", - "window": {"from": start.isoformat(), "to": end.isoformat()}, - "window_days": int(args.window_days), - "bulk_windows_total": len(all_windows), - "bulk_windows_done": windows_done, - "bulk_requests_this_run": requests_this_run, - "bulk_requests_logged_total": int(totals[0]), - "rows_upserted_this_run": rows_upserted, - "duplicate_rows_this_run": duplicate_rows, - "restated_rows_this_run": restated_rows, - "duplicate_rows_logged_total": int(totals[1]), - "restated_rows_logged_total": int(totals[2]), - "dedupe_policy": ( - "UNIQUE(symbol, announce_date); prefer more non-null fields, then " - "the provider's later occurrence; non-null bulk fields replace prior " - "values while null bulk fields retain existing values" - ), - "events_in_window": total_events, - "events_with_actual_and_estimate": paired_events, - "symbols_done": done_symbols, - "symbols_universe": len(symbols), - "announce_date_range": {"min": date_range[0], "max": date_range[1]}, - "request_budget": int(args.limit), - "stop_note": stop_note, - "complete": complete, - } - output = Path("reports/earnings-backfill-status.json") - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") - print(json.dumps(summary, indent=2)) - print(f"Wrote {output}") - - -if __name__ == "__main__": - asyncio.run(_main()) diff --git a/scripts/extend_snapshot_universe.py b/scripts/extend_snapshot_universe.py index 21a6ec8..ef74207 100644 --- a/scripts/extend_snapshot_universe.py +++ b/scripts/extend_snapshot_universe.py @@ -128,11 +128,10 @@ async def _resolve_pool() -> tuple[list[str], dict[str, str]]: """Return sorted unique symbols and source labels. Offline-safe: does **not** use production Postgres or SystemSetting cache - (those require a schema). Public sources first, then FMP, then seeds. + (those require a schema). Public sources first, then seeds. """ from app.services.ticker_universe_service import ( _SEED_UNIVERSES, - _fetch_universe_symbols_from_fmp, _fetch_universe_symbols_from_public, _normalise_symbols, ) @@ -150,19 +149,11 @@ async def _resolve_pool() -> tuple[list[str], dict[str, str]]: cleaned = _normalise_symbols(public_symbols) if cleaned: src = public_source or "public" - else: - if public_failures: - print( - f" WARNING: public fetch {universe}: " - f"{'; '.join(public_failures[:3])}" - ) - try: - fmp_symbols = await _fetch_universe_symbols_from_fmp(universe) - cleaned = _normalise_symbols(fmp_symbols) - if cleaned: - src = "fmp" - except Exception as exc: - print(f" WARNING: FMP fetch {universe}: {exc}") + elif public_failures: + print( + f" WARNING: public fetch {universe}: " + f"{'; '.join(public_failures[:3])}" + ) if not cleaned: cleaned = _normalise_symbols(_SEED_UNIVERSES.get(universe, [])) diff --git a/scripts/reparse_fundamentals.py b/scripts/reparse_fundamentals.py index f4fecc1..469117a 100644 --- a/scripts/reparse_fundamentals.py +++ b/scripts/reparse_fundamentals.py @@ -15,10 +15,10 @@ Cost: a reparse cannot be served from the database -- the facts a fixed parser n accepts were never stored -- so it refetches Company Facts for every tracked issuer under the SEC fair-access throttle. Expect a long run and a lot of network. -Scope note: this rewrites ``fundamental_snapshots`` only. As of the A5 gate those -rows feed the fundamentals API/UI and the parity report; scoring still reads the -legacy ``fundamental_data`` table, so a reparse does not move composite scores or -backtests until the cutover happens. +Scope note: this rewrites ``fundamental_snapshots`` only. Those rows now feed both +the fundamentals API/UI *and* — through the nightly ``fundamental_data`` refresh — +the fundamental dimension of the composite score, so a reparse does move scores +and backtests. Run it deliberately. Examples -------- diff --git a/scripts/run_tier1_macbook.sh b/scripts/run_tier1_macbook.sh index 26c3b18..52a81c2 100755 --- a/scripts/run_tier1_macbook.sh +++ b/scripts/run_tier1_macbook.sh @@ -3,7 +3,6 @@ # # Kept after Tier-1 cleanup: # --ssl-check diagnose corporate CA / proxy -# --earnings-only resume FMP earnings backfill + 2a/2b (parked) # --prod-book-matrix re-run 505 vs liquid universe × horizon book matrix # # Prerequisites: git checkout research branch, .env, deep research.sqlite for @@ -21,8 +20,6 @@ cd "$ROOT" RESEARCH_SNAP="${RESEARCH_SNAP:-backtest_snapshots/research.sqlite}" PROD_SNAP="${PROD_SNAP:-backtest_snapshots/prod.sqlite}" WORKERS="${WORKERS:-8}" -FMP_LIMIT="${FMP_LIMIT:-250}" -FMP_SLEEP="${FMP_SLEEP:-0.35}" PYTHON="${PYTHON:-python3}" USE_CORP_PROXY="${USE_CORP_PROXY:-0}" PHASE="" @@ -35,7 +32,6 @@ usage() { while [[ $# -gt 0 ]]; do case "$1" in --ssl-check) PHASE=ssl; shift ;; - --earnings-only) PHASE=earnings; shift ;; --prod-book-matrix) PHASE=prod_book; shift ;; --corp-proxy) USE_CORP_PROXY=1; shift ;; --workers) WORKERS="$2"; shift 2 ;; @@ -46,7 +42,7 @@ while [[ $# -gt 0 ]]; do done if [[ -z "$PHASE" ]]; then - echo "Pick a phase: --ssl-check | --earnings-only | --prod-book-matrix" >&2 + echo "Pick a phase: --ssl-check | --prod-book-matrix" >&2 usage 1 fi @@ -104,7 +100,6 @@ print(json.dumps(ssl_status(), indent=2)) print("bootstrap ->", bootstrap_ssl()) for url in ( "https://data.alpaca.markets/v2/stocks/SPY/bars?timeframe=1Day&limit=1", - "https://financialmodelingprep.com/stable/profile?symbol=AAPL", ): try: req = urllib.request.Request(url, headers={"User-Agent": "ssl-check"}) @@ -118,17 +113,6 @@ PY setup_ssl case "$PHASE" in ssl) ssl_check ;; - earnings) - need_file "$PROD_SNAP" - need_file "$RESEARCH_SNAP" - log "Earnings Task 2 bulk backfill + registered 2a/2b closeout" - "$PYTHON" scripts/backfill_earnings_events.py \ - --snapshot "$PROD_SNAP" --from-date 2016-01-04 --window-days 30 \ - --limit "$FMP_LIMIT" --sleep "$FMP_SLEEP" - "$PYTHON" scripts/run_earnings_research.py \ - --snapshot "$RESEARCH_SNAP" --universe-snapshot "$PROD_SNAP" \ - --earnings-snapshot "$PROD_SNAP" --workers "$WORKERS" --allow-spawn - ;; prod_book) need_file "$RESEARCH_SNAP" log "Production book universe × horizon matrix" diff --git a/tests/unit/test_activation_settings.py b/tests/unit/test_activation_settings.py index 1c8cc98..5a2f36f 100644 --- a/tests/unit/test_activation_settings.py +++ b/tests/unit/test_activation_settings.py @@ -8,9 +8,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.exceptions import ValidationError from app.services.admin_service import ( get_activation_config, - get_fundamentals_cutover_config, update_activation_config, - update_fundamentals_cutover_config, ) @@ -78,18 +76,3 @@ class TestActivationConfig: async def test_rejects_out_of_range_confidence(self, session: AsyncSession): with pytest.raises(ValidationError): await update_activation_config(session, {"min_confidence": 120.0}) - - -class TestFundamentalsCutoverConfig: - async def test_defaults_off_when_unset(self, session: AsyncSession): - assert await get_fundamentals_cutover_config(session) == {"enabled": False} - - async def test_round_trips_explicit_switch(self, session: AsyncSession): - assert await update_fundamentals_cutover_config(session, True) == { - "enabled": True - } - assert await get_fundamentals_cutover_config(session) == {"enabled": True} - - assert await update_fundamentals_cutover_config(session, False) == { - "enabled": False - } diff --git a/tests/unit/test_earnings_research.py b/tests/unit/test_earnings_research.py index cfff0b3..b878842 100644 --- a/tests/unit/test_earnings_research.py +++ b/tests/unit/test_earnings_research.py @@ -1,6 +1,5 @@ from datetime import date, timedelta -from scripts.backfill_earnings_events import _dedupe_bulk_rows, _windows from scripts.import_dolthub_earnings import _align_symbol from scripts.run_earnings_research import ( _analyse_2a_trades, @@ -41,42 +40,6 @@ def test_dolthub_alignment_allows_fiscal_period_label_after_announcement() -> No assert matches == [(0, 0), (1, 1)] -def test_bulk_windows_cover_range_without_overlap() -> None: - result = _windows(date(2020, 1, 1), date(2020, 1, 10), 4) - assert result == [ - (date(2020, 1, 1), date(2020, 1, 4)), - (date(2020, 1, 5), date(2020, 1, 8)), - (date(2020, 1, 9), date(2020, 1, 10)), - ] - - -def test_bulk_dedupe_prefers_more_complete_and_counts_restatement() -> None: - rows = [ - { - "symbol": "AAPL", - "announce_date": "2024-01-01", - "announce_time": None, - "eps_estimate": 1.0, - "eps_actual": 1.1, - "revenue_estimate": None, - "revenue_actual": None, - }, - { - "symbol": "AAPL", - "announce_date": "2024-01-01", - "announce_time": "amc", - "eps_estimate": 1.0, - "eps_actual": 1.2, - "revenue_estimate": 10.0, - "revenue_actual": 11.0, - }, - ] - deduped, duplicates, restated = _dedupe_bulk_rows(rows) - assert duplicates == 1 - assert restated == 1 - assert deduped == [rows[1]] - - def test_2a_uses_net_r_strict_hold_and_next_session_stop() -> None: calendar = [ date(2024, 1, 2), diff --git a/tests/unit/test_finnhub_provider.py b/tests/unit/test_finnhub_provider.py deleted file mode 100644 index ab5ec21..0000000 --- a/tests/unit/test_finnhub_provider.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Unit tests for FinnhubFundamentalProvider unit conversions.""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, patch - -import httpx -import pytest - -from app.providers.fundamentals_chain import FinnhubFundamentalProvider - - -def _mock_response(status_code: int, json_data: object = None) -> httpx.Response: - return httpx.Response( - status_code=status_code, - json=json_data if json_data is not None else {}, - request=httpx.Request("GET", "https://example.com"), - ) - - -@pytest.fixture -def provider() -> FinnhubFundamentalProvider: - return FinnhubFundamentalProvider(api_key="test-key") - - -@pytest.mark.asyncio -async def test_finnhub_market_cap_converted_from_millions_to_dollars(provider): - """Finnhub marketCapitalization is in millions — store absolute USD. - - SPCX-scale example: ~$1.8T → Finnhub reports 1_800_000 (millions). - Without conversion the UI showed 1.8M / micro cap. - """ - profile = {"marketCapitalization": 1_800_000} # millions → $1.8T - metrics = {"metric": {"peTTM": 40.0, "revenueGrowthTTMYoy": 25.0}} - earnings = [{"surprisePercent": 2.5}] - calendar = {"earningsCalendar": []} - - async def mock_get(url, params=None): - if "profile2" in url: - return _mock_response(200, profile) - if "stock/metric" in url: - return _mock_response(200, metrics) - if "stock/earnings" in url: - return _mock_response(200, earnings) - if "calendar/earnings" in url: - return _mock_response(200, calendar) - return _mock_response(200, {}) - - with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient: - instance = AsyncMock() - instance.get.side_effect = mock_get - instance.__aenter__ = AsyncMock(return_value=instance) - instance.__aexit__ = AsyncMock(return_value=False) - MockClient.return_value = instance - - result = await provider.fetch_fundamentals("SPCX") - - assert result.market_cap == 1_800_000 * 1_000_000 # $1.8T - assert result.pe_ratio == 40.0 - assert result.revenue_growth == 25.0 - assert result.earnings_surprise == 2.5 - - -@pytest.mark.asyncio -async def test_finnhub_market_cap_none_when_missing(provider): - profile: dict = {} - metrics = {"metric": {}} - earnings: list = [] - calendar = {"earningsCalendar": []} - - async def mock_get(url, params=None): - if "profile2" in url: - return _mock_response(200, profile) - if "stock/metric" in url: - return _mock_response(200, metrics) - if "stock/earnings" in url: - return _mock_response(200, earnings) - if "calendar/earnings" in url: - return _mock_response(200, calendar) - return _mock_response(200, {}) - - with patch("app.providers.fundamentals_chain.httpx.AsyncClient") as MockClient: - instance = AsyncMock() - instance.get.side_effect = mock_get - instance.__aenter__ = AsyncMock(return_value=instance) - instance.__aexit__ = AsyncMock(return_value=False) - MockClient.return_value = instance - - result = await provider.fetch_fundamentals("XYZ") - - assert result.market_cap is None - assert "market_cap" in result.unavailable_fields diff --git a/tests/unit/test_fmp_provider.py b/tests/unit/test_fmp_provider.py deleted file mode 100644 index 3cdaa92..0000000 --- a/tests/unit/test_fmp_provider.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Unit tests for FMPFundamentalProvider 402 reason recording.""" - -from __future__ import annotations - -from unittest.mock import AsyncMock, patch - -import httpx -import pytest - -from app.providers.fmp import FMPFundamentalProvider - - -def _mock_response(status_code: int, json_data: object = None) -> httpx.Response: - """Build a fake httpx.Response.""" - resp = httpx.Response( - status_code=status_code, - json=json_data if json_data is not None else {}, - request=httpx.Request("GET", "https://example.com"), - ) - return resp - - -@pytest.fixture -def provider() -> FMPFundamentalProvider: - return FMPFundamentalProvider(api_key="test-key") - - -class TestFetchJsonOptional402Tracking: - """_fetch_json_optional returns (data, was_402) tuple.""" - - @pytest.mark.asyncio - async def test_returns_empty_dict_and_true_on_402(self, provider): - mock_client = AsyncMock() - mock_client.get.return_value = _mock_response(402) - - data, was_402 = await provider._fetch_json_optional( - mock_client, "ratios-ttm", {}, "AAPL" - ) - - assert data == {} - assert was_402 is True - - @pytest.mark.asyncio - async def test_returns_data_and_false_on_200(self, provider): - mock_client = AsyncMock() - mock_client.get.return_value = _mock_response( - 200, [{"priceToEarningsRatioTTM": 25.5}] - ) - - data, was_402 = await provider._fetch_json_optional( - mock_client, "ratios-ttm", {}, "AAPL" - ) - - assert data == {"priceToEarningsRatioTTM": 25.5} - assert was_402 is False - - -class TestFetchFundamentals402Recording: - """fetch_fundamentals records 402 endpoints in unavailable_fields.""" - - @pytest.mark.asyncio - async def test_all_402_records_all_fields(self, provider): - """When all supplementary endpoints return 402, all three fields are recorded.""" - profile_resp = _mock_response(200, [{"marketCap": 1_000_000}]) - ratios_resp = _mock_response(402) - growth_resp = _mock_response(402) - earnings_resp = _mock_response(402) - - async def mock_get(url, params=None): - if "profile" in url: - return profile_resp - if "ratios-ttm" in url: - return ratios_resp - if "financial-growth" in url: - return growth_resp - if "earnings" in url: - return earnings_resp - return _mock_response(200, [{}]) - - with patch("app.providers.fmp.httpx.AsyncClient") as MockClient: - instance = AsyncMock() - instance.get.side_effect = mock_get - instance.__aenter__ = AsyncMock(return_value=instance) - instance.__aexit__ = AsyncMock(return_value=False) - MockClient.return_value = instance - - result = await provider.fetch_fundamentals("AAPL") - - assert result.unavailable_fields == { - "pe_ratio": "requires paid plan", - "revenue_growth": "requires paid plan", - "earnings_surprise": "requires paid plan", - } - - @pytest.mark.asyncio - async def test_mixed_200_402_records_only_402_fields(self, provider): - """When only ratios-ttm returns 402, only pe_ratio is recorded.""" - profile_resp = _mock_response(200, [{"marketCap": 2_000_000}]) - ratios_resp = _mock_response(402) - growth_resp = _mock_response(200, [{"revenueGrowth": 0.15}]) - earnings_resp = _mock_response(200, [{"epsActual": 3.0, "epsEstimated": 2.5}]) - - async def mock_get(url, params=None): - if "profile" in url: - return profile_resp - if "ratios-ttm" in url: - return ratios_resp - if "financial-growth" in url: - return growth_resp - if "earnings" in url: - return earnings_resp - return _mock_response(200, [{}]) - - with patch("app.providers.fmp.httpx.AsyncClient") as MockClient: - instance = AsyncMock() - instance.get.side_effect = mock_get - instance.__aenter__ = AsyncMock(return_value=instance) - instance.__aexit__ = AsyncMock(return_value=False) - MockClient.return_value = instance - - result = await provider.fetch_fundamentals("AAPL") - - assert result.unavailable_fields == {"pe_ratio": "requires paid plan"} - assert result.revenue_growth == 0.15 - assert result.earnings_surprise is not None - - @pytest.mark.asyncio - async def test_no_402_empty_unavailable_fields(self, provider): - """When all endpoints succeed, unavailable_fields is empty.""" - profile_resp = _mock_response(200, [{"marketCap": 3_000_000}]) - ratios_resp = _mock_response(200, [{"priceToEarningsRatioTTM": 20.0}]) - growth_resp = _mock_response(200, [{"revenueGrowth": 0.10}]) - earnings_resp = _mock_response(200, [{"epsActual": 2.0, "epsEstimated": 1.8}]) - - async def mock_get(url, params=None): - if "profile" in url: - return profile_resp - if "ratios-ttm" in url: - return ratios_resp - if "financial-growth" in url: - return growth_resp - if "earnings" in url: - return earnings_resp - return _mock_response(200, [{}]) - - with patch("app.providers.fmp.httpx.AsyncClient") as MockClient: - instance = AsyncMock() - instance.get.side_effect = mock_get - instance.__aenter__ = AsyncMock(return_value=instance) - instance.__aexit__ = AsyncMock(return_value=False) - MockClient.return_value = instance - - result = await provider.fetch_fundamentals("AAPL") - - assert result.unavailable_fields == {} - assert result.pe_ratio == 20.0 diff --git a/tests/unit/test_fundamental_data_refresh.py b/tests/unit/test_fundamental_data_refresh.py index 2508fd7..3b4ce82 100644 --- a/tests/unit/test_fundamental_data_refresh.py +++ b/tests/unit/test_fundamental_data_refresh.py @@ -15,7 +15,6 @@ from app.models.fundamental import FundamentalData from app.models.fundamental_snapshot import FundamentalSnapshot from app.models.ohlcv import OHLCVRecord from app.models.score import CompositeScore, DimensionScore -from app.models.settings import SystemSetting from app.models.ticker import Ticker from app.services import fundamentals_candidate_service as candidates from app.services import fundamentals_derivation as deriv @@ -76,49 +75,9 @@ def _snapshot_rows(cik: str) -> list[FundamentalSnapshot]: return rows -async def test_default_off_performs_no_candidate_read_or_write( - session: AsyncSession, monkeypatch -): - ticker = Ticker(symbol="AAA") - session.add(ticker) - await session.flush() - session.add( - FundamentalData( - ticker_id=ticker.id, - pe_ratio=12, - revenue_growth=3, - earnings_surprise=1, - market_cap=100, - fetched_at=NOW, - ) - ) - await session.commit() - - async def should_not_read(*args, **kwargs): - raise AssertionError("default-off refresh derived candidates") - - monkeypatch.setattr(candidates, "build_candidates", should_not_read) - summary = await refresh_service.refresh_if_enabled(session, today=TODAY) - - stored = await session.scalar( - select(FundamentalData).where(FundamentalData.ticker_id == ticker.id) - ) - assert summary == { - "enabled": False, - "refreshed": 0, - "score_inputs_changed": 0, - "dimension_scores_staled": 0, - "composite_scores_staled": 0, - } - assert stored.pe_ratio == 12 - - -async def test_activated_refresh_updates_all_fields_and_invalidates_scores( +async def test_refresh_updates_all_fields_and_invalidates_scores( session: AsyncSession, ): - session.add( - SystemSetting(key=refresh_service.ACTIVATION_KEY, value="true") - ) first = Ticker(symbol="AAA", cik="0000000001") second = Ticker(symbol="AAB", cik="0000000001") session.add_all([first, second]) @@ -191,7 +150,7 @@ async def test_activated_refresh_updates_all_fields_and_invalidates_scores( ) await session.commit() - summary = await refresh_service.refresh_if_enabled( + summary = await refresh_service.refresh( session, now=NOW, today=TODAY ) @@ -225,7 +184,7 @@ async def test_activated_refresh_updates_all_fields_and_invalidates_scores( for row in (*dimensions, *composites): row.is_stale = False await session.commit() - unchanged = await refresh_service.refresh_if_enabled( + unchanged = await refresh_service.refresh( session, now=NOW + timedelta(hours=1), today=TODAY ) assert unchanged["score_inputs_changed"] == 0 diff --git a/tests/unit/test_fundamental_service.py b/tests/unit/test_fundamental_service.py index 7fa7026..98f306a 100644 --- a/tests/unit/test_fundamental_service.py +++ b/tests/unit/test_fundamental_service.py @@ -1,13 +1,20 @@ -"""Unit tests for fundamental_service — unavailable_fields persistence.""" +"""Unit tests for fundamental_service — the surviving read path. + +Writes to ``fundamental_data`` are covered by test_fundamental_data_refresh.py; +this file only guards the lookup used by the router and scoring. +""" from __future__ import annotations import json +from datetime import datetime, timezone import pytest from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from app.database import Base +from app.exceptions import NotFoundError +from app.models.fundamental import FundamentalData from app.models.ticker import Ticker from app.services import fundamental_service @@ -43,57 +50,36 @@ async def ticker(session: AsyncSession) -> Ticker: @pytest.mark.asyncio -async def test_store_fundamental_persists_unavailable_fields( +async def test_get_fundamental_returns_the_cached_row( session: AsyncSession, ticker: Ticker ): - """unavailable_fields dict is serialized to JSON and stored.""" - fields = {"pe_ratio": "requires paid plan", "revenue_growth": "requires paid plan"} - - record = await fundamental_service.store_fundamental( - session, - symbol="AAPL", - pe_ratio=None, - revenue_growth=None, - market_cap=1_000_000.0, - unavailable_fields=fields, + fields = {"pe_ratio": "split guard applied"} + session.add( + FundamentalData( + ticker_id=ticker.id, + pe_ratio=None, + market_cap=1_000_000.0, + fetched_at=datetime.now(timezone.utc), + unavailable_fields_json=json.dumps(fields), + ) ) + await session.commit() + record = await fundamental_service.get_fundamental(session, symbol="aapl") + + assert record is not None + assert record.market_cap == 1_000_000.0 assert json.loads(record.unavailable_fields_json) == fields @pytest.mark.asyncio -async def test_store_fundamental_defaults_to_empty_dict( +async def test_get_fundamental_returns_none_without_a_cached_row( session: AsyncSession, ticker: Ticker ): - """When unavailable_fields is not provided, column defaults to '{}'.""" - record = await fundamental_service.store_fundamental( - session, - symbol="AAPL", - pe_ratio=25.0, - ) - - assert json.loads(record.unavailable_fields_json) == {} + assert await fundamental_service.get_fundamental(session, symbol="AAPL") is None @pytest.mark.asyncio -async def test_store_fundamental_updates_unavailable_fields( - session: AsyncSession, ticker: Ticker -): - """Updating an existing record also updates unavailable_fields_json.""" - # First store - await fundamental_service.store_fundamental( - session, - symbol="AAPL", - pe_ratio=None, - unavailable_fields={"pe_ratio": "requires paid plan"}, - ) - - # Second store — fields now available - record = await fundamental_service.store_fundamental( - session, - symbol="AAPL", - pe_ratio=25.0, - unavailable_fields={}, - ) - - assert json.loads(record.unavailable_fields_json) == {} +async def test_get_fundamental_rejects_an_unknown_symbol(session: AsyncSession): + with pytest.raises(NotFoundError): + await fundamental_service.get_fundamental(session, symbol="NOPE") diff --git a/tests/unit/test_fundamentals_chain_provider.py b/tests/unit/test_fundamentals_chain_provider.py deleted file mode 100644 index 6bda245..0000000 --- a/tests/unit/test_fundamentals_chain_provider.py +++ /dev/null @@ -1,181 +0,0 @@ -"""Unit tests for chained fundamentals provider fallback behavior.""" - -from __future__ import annotations - -from datetime import datetime, timezone - -import pytest - -from app.exceptions import ProviderError, RateLimitError -from app.providers.fundamentals_chain import ChainedFundamentalProvider -from app.providers.protocol import FundamentalData - - -class _FailProvider: - def __init__(self, message: str) -> None: - self._message = message - - async def fetch_fundamentals(self, ticker: str) -> FundamentalData: - raise ProviderError(f"{self._message} ({ticker})") - - -class _RateLimitedProvider: - async def fetch_fundamentals(self, ticker: str) -> FundamentalData: - raise RateLimitError(f"rate limit hit for {ticker}") - - -class _DataProvider: - def __init__(self, data: FundamentalData) -> None: - self._data = data - - async def fetch_fundamentals(self, ticker: str) -> FundamentalData: - return FundamentalData( - ticker=ticker, - pe_ratio=self._data.pe_ratio, - revenue_growth=self._data.revenue_growth, - earnings_surprise=self._data.earnings_surprise, - market_cap=self._data.market_cap, - fetched_at=self._data.fetched_at, - unavailable_fields=self._data.unavailable_fields, - ) - - -@pytest.mark.asyncio -async def test_chained_provider_uses_fallback_provider_on_primary_failure(): - fallback_data = FundamentalData( - ticker="AAPL", - pe_ratio=25.0, - revenue_growth=None, - earnings_surprise=None, - market_cap=1_000_000.0, - fetched_at=datetime.now(timezone.utc), - unavailable_fields={}, - ) - - provider = ChainedFundamentalProvider([ - ("primary", _FailProvider("primary down")), - ("fallback", _DataProvider(fallback_data)), - ]) - - result = await provider.fetch_fundamentals("AAPL") - - assert result.pe_ratio == 25.0 - assert result.market_cap == 1_000_000.0 - assert result.unavailable_fields.get("source_pe_ratio") == "fallback" - - -@pytest.mark.asyncio -async def test_chained_provider_merges_fields_across_providers(): - """Primary supplies only market cap; fallback fills P/E and earnings.""" - primary_data = FundamentalData( - ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None, - market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={}, - ) - fallback_data = FundamentalData( - ticker="AAPL", pe_ratio=18.0, revenue_growth=12.0, earnings_surprise=4.0, - market_cap=999.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={}, - ) - - provider = ChainedFundamentalProvider([ - ("fmp", _DataProvider(primary_data)), - ("finnhub", _DataProvider(fallback_data)), - ]) - - result = await provider.fetch_fundamentals("AAPL") - - # market cap from primary (first to supply it), the rest from fallback - assert result.market_cap == 2_000_000.0 - assert result.pe_ratio == 18.0 - assert result.revenue_growth == 12.0 - assert result.earnings_surprise == 4.0 - assert result.unavailable_fields.get("source_market_cap") == "fmp" - assert result.unavailable_fields.get("source_pe_ratio") == "finnhub" - - -@pytest.mark.asyncio -async def test_chained_provider_raises_when_all_providers_fail(): - provider = ChainedFundamentalProvider([ - ("p1", _FailProvider("p1 failed")), - ("p2", _FailProvider("p2 failed")), - ]) - - with pytest.raises(ProviderError) as exc: - await provider.fetch_fundamentals("MSFT") - - assert "All fundamentals providers failed" in str(exc.value) - - -@pytest.mark.asyncio -async def test_rate_limited_fallback_raises_when_incomplete(): - """FMP gives market cap; the fallback is rate-limited → chain signals it so - the collector can back off instead of storing a degraded record.""" - primary_data = FundamentalData( - ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None, - market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={}, - ) - provider = ChainedFundamentalProvider([ - ("fmp", _DataProvider(primary_data)), - ("finnhub", _RateLimitedProvider()), - ]) - - with pytest.raises(RateLimitError): - await provider.fetch_fundamentals("AAPL") - - -@pytest.mark.asyncio -async def test_rate_limited_fallback_allows_partial(): - """With allow_partial=True the chain returns the market cap it did get.""" - primary_data = FundamentalData( - ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None, - market_cap=2_000_000.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={}, - ) - provider = ChainedFundamentalProvider([ - ("fmp", _DataProvider(primary_data)), - ("finnhub", _RateLimitedProvider()), - ]) - - result = await provider.fetch_fundamentals("AAPL", allow_partial=True) - assert result.market_cap == 2_000_000.0 - assert result.pe_ratio is None - - -@pytest.mark.asyncio -async def test_rate_limited_but_complete_does_not_raise(): - """If every field is filled, a rate limit on a later (unused) provider is moot.""" - full = FundamentalData( - ticker="AAPL", pe_ratio=20.0, revenue_growth=10.0, earnings_surprise=2.0, - market_cap=5.0, fetched_at=datetime.now(timezone.utc), unavailable_fields={}, - ) - provider = ChainedFundamentalProvider([ - ("fmp", _DataProvider(full)), - ("finnhub", _RateLimitedProvider()), - ]) - - result = await provider.fetch_fundamentals("AAPL") - assert result.pe_ratio == 20.0 - - -@pytest.mark.asyncio -async def test_chain_merges_next_earnings_date(): - """Earnings date is taken from the first provider that supplies it.""" - from datetime import date as _date - - primary = FundamentalData( - ticker="AAPL", pe_ratio=None, revenue_growth=None, earnings_surprise=None, - market_cap=100.0, fetched_at=datetime.now(timezone.utc), - ) - - class _EarningsProvider: - async def fetch_fundamentals(self, ticker: str) -> FundamentalData: - return FundamentalData( - ticker=ticker, pe_ratio=10.0, revenue_growth=5.0, earnings_surprise=1.0, - market_cap=None, fetched_at=datetime.now(timezone.utc), - next_earnings_date=_date(2026, 7, 1), - ) - - provider = ChainedFundamentalProvider([ - ("fmp", _DataProvider(primary)), - ("finnhub", _EarningsProvider()), - ]) - result = await provider.fetch_fundamentals("AAPL") - assert result.next_earnings_date == _date(2026, 7, 1) diff --git a/tests/unit/test_fundamentals_parity.py b/tests/unit/test_fundamentals_parity.py deleted file mode 100644 index a0495ae..0000000 --- a/tests/unit/test_fundamentals_parity.py +++ /dev/null @@ -1,209 +0,0 @@ -"""A5 fundamentals parity report: read-only comparison + artifact archive.""" - -from __future__ import annotations - -from datetime import date, datetime, timezone - -import pytest -from sqlalchemy import func, select - -from app.models.data_import_run import DataImportRun -from app.models.earnings_event import EarningsEvent -from app.models.fundamental import FundamentalData -from app.models.fundamental_snapshot import FundamentalSnapshot -from app.models.ohlcv import OHLCVRecord -from app.models.ticker import Ticker -from app.services.fundamentals_parity_service import ( - build_report, - fundamental_score, - load_latest, - load_latest_csv, - load_latest_json, - store_report, -) - -UTC = timezone.utc -GENERATED = datetime(2026, 7, 23, 10, 30, tzinfo=UTC) - - -def _snapshot_rows(cik: str) -> list[FundamentalSnapshot]: - rows = [] - periods = ("Q1", "Q2", "Q3", "FY") - months = (3, 6, 9, 12) - for fy, multiplier in ((2025, 1.0), (2026, 1.1)): - revenues = [100 * multiplier, 110 * multiplier, 120 * multiplier, 130 * multiplier] - eps = [1.0 * multiplier, 1.1 * multiplier, 1.2 * multiplier, 1.3 * multiplier] - for index, period in enumerate(periods): - period_end = date(fy, months[index], 28) - rows.append( - FundamentalSnapshot( - cik=cik, - accession=f"{cik}-{fy}-{period}", - form="10-K" if period == "FY" else "10-Q", - filed_date=period_end, - accepted_at=datetime(fy, months[index], 28, tzinfo=UTC), - period_end=period_end, - fiscal_year=fy, - fiscal_period=period, - revenue=sum(revenues[: index + 1]), - operating_income=sum(revenues[: index + 1]) * 0.2, - diluted_eps=sum(eps[: index + 1]), - cfo=sum(revenues[: index + 1]) * 0.25, - capex=sum(revenues[: index + 1]) * 0.05, - depreciation_amortization=sum(revenues[: index + 1]) * 0.05, - cash_and_st_investments=40, - total_debt=100, - shares_outstanding=1000, - ) - ) - return rows - - -async def _seed(db_session): - first = Ticker(symbol="AAA", cik="0000000001", sic="3571") - second = Ticker(symbol="BBB", cik=None, sic=None) - db_session.add_all([first, second]) - await db_session.flush() - db_session.add_all(_snapshot_rows(first.cik)) - db_session.add_all( - [ - FundamentalData( - ticker_id=first.id, - pe_ratio=25, - revenue_growth=5, - earnings_surprise=0, - fetched_at=GENERATED, - ), - FundamentalData( - ticker_id=second.id, - pe_ratio=12, - revenue_growth=3, - earnings_surprise=None, - fetched_at=GENERATED, - ), - OHLCVRecord( - ticker_id=first.id, - date=date(2026, 7, 22), - open=100, - high=100, - low=100, - close=100, - volume=100, - ), - EarningsEvent( - ticker_id=first.id, - announce_date=date(2026, 7, 1), - session="amc", - eps_estimate=2, - eps_actual=2.2, - source="dolt_earnings", - ), - DataImportRun( - source="sec_facts", - revision="sec-rev", - status="promoted", - source_max_date=date(2026, 7, 22), - started_at=GENERATED, - completed_at=GENERATED, - ), - DataImportRun( - source="dolt_earnings", - revision="dolt-rev", - status="no_op", - source_max_date=date(2026, 7, 22), - started_at=GENERATED, - completed_at=GENERATED, - ), - ] - ) - await db_session.flush() - - -def test_score_formula_matches_production_rules(): - score = fundamental_score(pe_ratio=15, revenue_growth=0, earnings_surprise=0) - assert score == pytest.approx((100 + 50 + 50) / 3) - assert fundamental_score(pe_ratio=15, revenue_growth=None, earnings_surprise=None) is None - -async def test_report_compares_sources_and_leaves_database_untouched(db_session): - await _seed(db_session) - before = await db_session.scalar(select(func.count()).select_from(FundamentalData)) - - report = await build_report( - db_session, - generated_at=GENERATED, - today=date(2026, 7, 23), - ) - - after = await db_session.scalar(select(func.count()).select_from(FundamentalData)) - assert before == after == 2 - assert not db_session.new and not db_session.dirty and not db_session.deleted - assert report["read_only"] is True - assert report["approval_status"] == "pending_explicit_approval" - assert report["source_runs"]["sec_facts"]["revision"] == "sec-rev" - assert report["source_runs"]["dolt_earnings"]["revision"] == "dolt-rev" - - first = next(row for row in report["rows"] if row["symbol"] == "AAA") - assert first["fields"]["pe_ratio"]["candidate"] == pytest.approx( - 100 / 5.06, abs=1e-4 - ) - assert first["fields"]["revenue_growth"]["candidate"] == pytest.approx(10) - assert first["fields"]["earnings_surprise"]["candidate"] == pytest.approx(10) - assert first["scores"]["candidate_fundamental"] is not None - assert report["summary"]["universe_count"] == 2 - assert report["summary"]["field_stats"]["pe_ratio"]["both_available"] == 1 - - -async def test_artifacts_archive_and_latest_manifest(db_session, tmp_path): - await _seed(db_session) - report = await build_report( - db_session, - generated_at=GENERATED, - today=date(2026, 7, 23), - ) - - paths = store_report(report, tmp_path) - - assert tmp_path.joinpath("latest.json").exists() - assert paths["json"].endswith(".json") and paths["csv"].endswith(".csv") - assert load_latest(tmp_path)["generated_at"] == GENERATED.isoformat() - csv_artifact = load_latest_csv(tmp_path) - assert csv_artifact is not None - assert csv_artifact[0].endswith(".csv") - assert "legacy_fundamental,candidate_fundamental" in csv_artifact[1] - assert "AAA" in csv_artifact[1] - json_artifact = load_latest_json(tmp_path) - assert json_artifact is not None and '"rows"' in json_artifact[1] - - -async def test_admin_endpoints_return_compact_summary_and_downloads( - client, db_session, tmp_path, monkeypatch -): - from app.config import settings - from app.dependencies import require_admin - from app.main import app - - await _seed(db_session) - report = await build_report( - db_session, - generated_at=GENERATED, - today=date(2026, 7, 23), - ) - store_report(report, tmp_path) - monkeypatch.setattr(settings, "fundamentals_parity_report_dir", str(tmp_path)) - app.dependency_overrides[require_admin] = lambda: None - try: - summary_response = await client.get("/api/v1/admin/fundamentals-parity") - assert summary_response.status_code == 200 - summary = summary_response.json()["data"] - assert summary["summary"]["universe_count"] == 2 - assert "rows" not in summary - - csv_response = await client.get("/api/v1/admin/fundamentals-parity/csv") - assert csv_response.status_code == 200 - assert "AAA" in csv_response.json()["data"]["content"] - - json_response = await client.get("/api/v1/admin/fundamentals-parity/json") - assert json_response.status_code == 200 - assert '"rows"' in json_response.json()["data"]["content"] - finally: - app.dependency_overrides.pop(require_admin, None) diff --git a/tests/unit/test_fundamentals_quality_service.py b/tests/unit/test_fundamentals_quality_service.py index 96ea3ab..c9533c8 100644 --- a/tests/unit/test_fundamentals_quality_service.py +++ b/tests/unit/test_fundamentals_quality_service.py @@ -6,7 +6,6 @@ from datetime import date, datetime, timezone from app.models.data_import_run import DataImportRun from app.models.fundamental_snapshot import FundamentalSnapshot from app.models.sec_filing_gap import SecFilingGap -from app.models.settings import SystemSetting from app.models.ticker import Ticker from app.services import fundamentals_quality_service @@ -19,12 +18,6 @@ async def test_latest_sec_validation_blocks_deferred_and_no_history_ciks( healthy = Ticker(symbol="HEALTHY", cik="0000000003") db_session.add_all([missing, no_history, healthy]) await db_session.flush() - db_session.add( - SystemSetting( - key="fundamental_data_sec_dolt_cutover_enabled", - value="true", - ) - ) db_session.add( DataImportRun( source="sec_facts", @@ -44,38 +37,11 @@ async def test_latest_sec_validation_blocks_deferred_and_no_history_ciks( } -async def test_sec_quality_gate_is_inactive_before_cutover(db_session): - ticker = Ticker(symbol="SHADOW", cik="0000000042") - db_session.add(ticker) - await db_session.flush() - now = datetime.now(timezone.utc) - db_session.add( - SecFilingGap( - cik=ticker.cik, - accession="SHADOW-Q", - form="10-Q", - index_date=date.today(), - reason="not_in_companyfacts", - first_seen_at=now, - last_attempted_at=now, - ) - ) - await db_session.flush() - - assert await fundamentals_quality_service.blocked_ticker_ids(db_session) == set() - - - - async def test_active_gap_is_blocked_until_a_later_filing_supersedes_it(db_session): ticker = Ticker(symbol="HIST", cik="0000000043") now = datetime.now(timezone.utc) db_session.add_all([ ticker, - SystemSetting( - key="fundamental_data_sec_dolt_cutover_enabled", - value="true", - ), SecFilingGap( cik=ticker.cik, accession="HIST-Q", @@ -116,10 +82,6 @@ async def test_gap_without_index_date_uses_first_seen_date_for_supersession( first_seen = datetime(2026, 5, 1, 12, tzinfo=timezone.utc) db_session.add_all([ ticker, - SystemSetting( - key="fundamental_data_sec_dolt_cutover_enabled", - value="true", - ), SecFilingGap( cik=ticker.cik, accession="DATELESS-Q", @@ -157,10 +119,6 @@ async def test_ticker_quality_explains_no_xbrl_block(db_session): ticker = Ticker(symbol="NEWREG", cik="0000000044") db_session.add_all([ ticker, - SystemSetting( - key="fundamental_data_sec_dolt_cutover_enabled", - value="true", - ), DataImportRun( source="sec_facts", status="promoted", diff --git a/tests/unit/test_ingestion_fundamentals_source.py b/tests/unit/test_ingestion_fundamentals_source.py new file mode 100644 index 0000000..bb01168 --- /dev/null +++ b/tests/unit/test_ingestion_fundamentals_source.py @@ -0,0 +1,38 @@ +"""A6: `sources=fundamentals` is accepted but never fetches from a provider. + +`fundamental_data` is rebuilt for the whole universe by the nightly SEC + Dolt +imports, so there is no per-ticker fetch left. The source key stays valid so an +older client gets a truthful `skipped` instead of a silent omission. +""" + +from __future__ import annotations + +from app.models.ticker import Ticker + + +async def test_fundamentals_source_reports_skipped(client, db_session): + from app.dependencies import require_access + from app.main import app + + app.dependency_overrides[require_access] = lambda: None + try: + db_session.add(Ticker(symbol="AAPL")) + await db_session.flush() + + resp = await client.post( + "/api/v1/ingestion/fetch/AAPL", params={"sources": "fundamentals"} + ) + assert resp.status_code == 200 + source = resp.json()["data"]["sources"]["fundamentals"] + assert source["status"] == "skipped" + assert "SEC + Dolt" in source["message"] + finally: + app.dependency_overrides.pop(require_access, None) + + +def test_fundamentals_remains_a_recognised_source_key(): + """Older clients keep getting an entry for it rather than a missing key.""" + from app.routers.ingestion import _parse_requested_sources + + assert "fundamentals" in _parse_requested_sources("fundamentals") + assert "fundamentals" in _parse_requested_sources(None) # None => all sources diff --git a/tests/unit/test_rr_scanner_preservation.py b/tests/unit/test_rr_scanner_preservation.py index b9fba53..dd76db1 100644 --- a/tests/unit/test_rr_scanner_preservation.py +++ b/tests/unit/test_rr_scanner_preservation.py @@ -24,7 +24,6 @@ from app.models.ohlcv import OHLCVRecord from app.models.paper_trade import PaperTrade from app.models.signal_context_snapshot import SignalContextSnapshot from app.models.sec_filing_gap import SecFilingGap -from app.models.settings import SystemSetting from app.models.sr_level import SRLevel from app.models.ticker import Ticker from app.models.trade_setup import TradeSetup @@ -524,10 +523,6 @@ async def test_get_trade_setups_hides_active_sec_filing_gap( db_session.add(ticker) await db_session.flush() db_session.add_all([ - SystemSetting( - key="fundamental_data_sec_dolt_cutover_enabled", - value="true", - ), SecFilingGap( cik=ticker.cik, accession="0000000042-26-000001", diff --git a/tests/unit/test_schedule_config.py b/tests/unit/test_schedule_config.py index 211edd3..db1a53a 100644 --- a/tests/unit/test_schedule_config.py +++ b/tests/unit/test_schedule_config.py @@ -66,26 +66,11 @@ class TestTradingDayCrons: assert "Mon" in weekdays, f"{key} skips Mondays — numeric day-of-week?" assert {"Sat", "Sun"}.isdisjoint(weekdays), f"{key} fires on a weekend" - def test_fundamentals_runs_on_monday(self): - from datetime import datetime - - from apscheduler.triggers.cron import CronTrigger - - trigger = CronTrigger.from_crontab( - SCHEDULE_DEFAULTS["schedule_fundamentals_cron"], - timezone=SCHEDULE_DEFAULTS["schedule_timezone"], - ) - fire = trigger.get_next_fire_time( - None, datetime(2026, 7, 19, tzinfo=trigger.timezone) - ) - assert fire.strftime("%a") == "Mon" - @pytest.mark.parametrize( ("key", "hour", "minute"), ( ("schedule_dolt_earnings_cron", 2, 30), ("schedule_sec_fundamentals_cron", 4, 0), - ("schedule_fundamentals_parity_cron", 5, 30), ), ) def test_shadow_imports_run_daily_at_expected_et_time( @@ -122,7 +107,7 @@ class TestScheduleConfig: async def test_rejects_bad_cron(self, session: AsyncSession): with pytest.raises(ValidationError): - await update_schedule_config(session, {"schedule_fundamentals_cron": "every monday"}) + await update_schedule_config(session, {"schedule_daily_pipeline_cron": "every monday"}) async def test_rejects_bad_timezone(self, session: AsyncSession): with pytest.raises(ValidationError): diff --git a/tests/unit/test_scheduler.py b/tests/unit/test_scheduler.py index e63359a..853789e 100644 --- a/tests/unit/test_scheduler.py +++ b/tests/unit/test_scheduler.py @@ -13,8 +13,6 @@ from app.scheduler import ( _resume_tickers, _last_successful, _run_shadow_import, - collect_fundamentals, - run_fundamentals_parity_report, run_sec_fundamentals_import, configure_scheduler, get_job_runtime_snapshot, @@ -123,10 +121,8 @@ class TestConfigureScheduler: "data_backfill", "benchmark_collector", "sentiment_collector", - "fundamental_collector", "dolt_earnings_import", "sec_fundamentals_import", - "fundamentals_parity_report", "rr_scanner", "shadow_book", "ticker_universe_sync", @@ -157,10 +153,8 @@ class TestConfigureScheduler: "intraday_pipeline", "data_collector", "data_backfill", - "fundamental_collector", "dolt_earnings_import", "sec_fundamentals_import", - "fundamentals_parity_report", "market_regime", "near_close_pipeline", "regime_monitor", @@ -181,41 +175,6 @@ class _SessionContext: return None -class TestFundamentalCollector: - @staticmethod - def _session_factory(): - return _SessionContext() - - async def test_skips_legacy_provider_when_cutover_is_active(self, monkeypatch): - async def enabled(db, job_name): - return True - - async def cutover_enabled(db): - return True - - async def unexpected_ticker_lookup(db): - raise AssertionError("legacy ticker lookup must not run after cutover") - - monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory) - monkeypatch.setattr("app.scheduler._is_job_enabled", enabled) - monkeypatch.setattr( - "app.scheduler.fundamental_data_refresh_service.is_enabled", - cutover_enabled, - ) - monkeypatch.setattr( - "app.scheduler._get_fundamental_priority_tickers", - unexpected_ticker_lookup, - ) - - await collect_fundamentals() - - runtime = get_job_runtime_snapshot("fundamental_collector") - assert runtime["status"] == "skipped" - assert runtime["processed"] == 0 - assert runtime["total"] == 0 - assert runtime["message"] == "SEC + Dolt fundamentals cutover is active" - - class TestShadowImportJobs: @staticmethod def _session_factory(): @@ -329,7 +288,6 @@ class TestShadowImportJobs: async def refreshed(db): calls.append(db) return { - "enabled": True, "refreshed": 511, "score_inputs_changed": 2, "dimension_scores_staled": 2, @@ -340,7 +298,7 @@ class TestShadowImportJobs: monkeypatch.setattr("app.scheduler._is_job_enabled", enabled) monkeypatch.setattr("app.scheduler.run_import", unavailable) monkeypatch.setattr( - "app.scheduler.fundamental_data_refresh_service.refresh_if_enabled", + "app.scheduler.fundamental_data_refresh_service.refresh", refreshed, ) @@ -351,7 +309,7 @@ class TestShadowImportJobs: assert runtime["status"] == "error" assert runtime["message"] == "SEC unavailable" - async def test_sec_success_surfaces_activated_refresh_summary(self, monkeypatch): + async def test_sec_success_surfaces_cache_refresh_summary(self, monkeypatch): async def enabled(db, job_name): return True @@ -362,7 +320,6 @@ class TestShadowImportJobs: async def refreshed(db): return { - "enabled": True, "refreshed": 511, "score_inputs_changed": 2, "dimension_scores_staled": 2, @@ -373,7 +330,7 @@ class TestShadowImportJobs: monkeypatch.setattr("app.scheduler._is_job_enabled", enabled) monkeypatch.setattr("app.scheduler.run_import", imported) monkeypatch.setattr( - "app.scheduler.fundamental_data_refresh_service.refresh_if_enabled", + "app.scheduler.fundamental_data_refresh_service.refresh", refreshed, ) @@ -385,52 +342,45 @@ class TestShadowImportJobs: "no_op · abcdef123456 · cache 511 · 2 score inputs changed" ) - async def test_disabled_sec_job_does_not_run_local_refresh(self, monkeypatch): + async def test_disabled_sec_job_still_refreshes_local_cache(self, monkeypatch): + """Disabling the job stops the SEC fetch, not the local cache. + + The cache is derived from stored snapshots, earnings events and closes. + Prices and earnings move daily even when no filing does, and there is no + provider fallback since A6 — freezing it would silently stale scoring. + """ + calls = [] + async def disabled(db, job_name): return False async def should_not_run(*args, **kwargs): - raise AssertionError("disabled SEC job ran work") + raise AssertionError("disabled SEC job hit the network") + + async def refreshed(db): + calls.append(db) + return { + "refreshed": 511, + "score_inputs_changed": 2, + "dimension_scores_staled": 2, + "composite_scores_staled": 2, + } monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory) monkeypatch.setattr("app.scheduler._is_job_enabled", disabled) monkeypatch.setattr("app.scheduler.run_import", should_not_run) monkeypatch.setattr( - "app.scheduler.fundamental_data_refresh_service.refresh_if_enabled", - should_not_run, + "app.scheduler.fundamental_data_refresh_service.refresh", + refreshed, ) await run_sec_fundamentals_import() + assert len(calls) == 1 runtime = get_job_runtime_snapshot("sec_fundamentals_import") - assert runtime["status"] == "skipped" - assert runtime["message"] == "Disabled" - - -async def test_fundamentals_parity_job_surfaces_report_summary(monkeypatch): - async def enabled(db, job_name): - return True - - async def generated(db, report_dir): - return ( - { - "generated_at": "2026-07-23T10:30:00+00:00", - "summary": { - "universe_count": 511, - "fundamental_score_material_changes": 12, - }, - }, - {"json": "report.json", "csv": "report.csv"}, + assert runtime["status"] == "completed" + assert runtime["message"] == ( + "Import disabled · cache 511 · 2 score inputs changed" ) - monkeypatch.setattr("app.scheduler.async_session_factory", TestShadowImportJobs._session_factory) - monkeypatch.setattr("app.scheduler._is_job_enabled", enabled) - monkeypatch.setattr( - "app.scheduler.fundamentals_parity_service.generate_and_store", generated - ) - await run_fundamentals_parity_report() - - runtime = get_job_runtime_snapshot("fundamentals_parity_report") - assert runtime["status"] == "completed" - assert runtime["message"] == "511 tickers · 12 material score changes" diff --git a/tests/unit/test_ticker_universe_service.py b/tests/unit/test_ticker_universe_service.py index b17d216..d3b3937 100644 --- a/tests/unit/test_ticker_universe_service.py +++ b/tests/unit/test_ticker_universe_service.py @@ -10,7 +10,6 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from app.database import Base -from app.exceptions import ProviderError from app.models.settings import SystemSetting from app.models.ticker import Ticker from app.services import ticker_universe_service @@ -97,11 +96,7 @@ async def test_fetch_universe_symbols_uses_cached_snapshot_when_live_sources_fai async def _fake_public(_universe: str): return [], ["public failed"], None - async def _fake_fmp(_universe: str): - raise ProviderError("fmp failed") - monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public) - monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp) symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500") assert symbols == ["AAPL", "MSFT"] @@ -116,11 +111,7 @@ async def test_fetch_universe_symbols_uses_seed_when_live_and_cache_fail( async def _fake_public(_universe: str): return [], ["public failed"], None - async def _fake_fmp(_universe: str): - raise ProviderError("fmp failed") - monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_public", _fake_public) - monkeypatch.setattr(ticker_universe_service, "_fetch_universe_symbols_from_fmp", _fake_fmp) symbols, source = await ticker_universe_service.fetch_universe_symbols(session, "sp500") assert "AAPL" in symbols