Implement A5 fundamentals cutover activation

app/services/admin_service.py

@@ -637,7 +637,7 @@ JOB_LABELS = {
     "sentiment_collector": "Sentiment Collector",
     "fundamental_collector": "Fundamental Collector",
     "dolt_earnings_import": "Dolt Earnings Import (shadow)",
-    "sec_fundamentals_import": "SEC Fundamentals Import (shadow)",
+    "sec_fundamentals_import": "SEC Fundamentals Import",
     "fundamentals_parity_report": "Fundamentals Parity Report (read-only)",
     "rr_scanner": "R:R Scanner",
     "ticker_universe_sync": "Ticker Universe Sync",

app/services/fundamental_data_refresh_service.py

@@ -0,0 +1,179 @@
+"""A5 activation: refresh the legacy fundamentals cache from local bulk data."""
+
+from __future__ import annotations
+
+import json
+from datetime import date, datetime, timezone
+from typing import Any
+
+from sqlalchemy import select, update
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.database import insert_for_session
+from app.models.fundamental import FundamentalData
+from app.models.score import CompositeScore, DimensionScore
+from app.services import fundamentals_candidate_service, settings_store
+
+
+# Absence is deliberately false. Production activation therefore requires one
+# explicit, durable SystemSetting change after the A5 evidence is approved.
+ACTIVATION_KEY = "fundamental_data_sec_dolt_cutover_enabled"
+_SCORE_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise")
+
+
+async def is_enabled(db: AsyncSession) -> bool:
+    raw = await settings_store.get_value(db, ACTIVATION_KEY, "false")
+    return str(raw).strip().lower() == "true"
+
+
+async def refresh_if_enabled(
+    db: AsyncSession,
+    *,
+    now: datetime | None = None,
+    today: date | None = None,
+) -> dict[str, Any]:
+    """Refresh atomically when activated; otherwise perform no writes."""
+    if not await is_enabled(db):
+        return {
+            "enabled": False,
+            "refreshed": 0,
+            "score_inputs_changed": 0,
+            "dimension_scores_staled": 0,
+            "composite_scores_staled": 0,
+        }
+    return await refresh(db, now=now, today=today)
+
+
+async def refresh(
+    db: AsyncSession,
+    *,
+    now: datetime | None = None,
+    today: date | None = None,
+) -> dict[str, Any]:
+    """Replace every ticker's compat-cache row in one database transaction.
+
+    Candidate values are assembled before the first write and use only local
+    PostgreSQL tables. A failure rolls the whole refresh back. Only changes to
+    the three scoring inputs invalidate cached scores; market cap and the next
+    earnings date are display-only.
+    """
+    refreshed_at = now or datetime.now(timezone.utc)
+    candidates = await fundamentals_candidate_service.build_candidates(
+        db, today=today
+    )
+    ticker_ids = [candidate.ticker_id for candidate in candidates]
+    existing = await _existing_by_ticker(db, ticker_ids)
+    changed_ids = {
+        candidate.ticker_id
+        for candidate in candidates
+        if _score_inputs_changed(existing.get(candidate.ticker_id), candidate)
+    }
+
+    for candidate in candidates:
+        unavailable_json = json.dumps(
+            candidate.unavailable_fields, sort_keys=True
+        )
+        stmt = insert_for_session(db, FundamentalData).values(
+            ticker_id=candidate.ticker_id,
+            pe_ratio=candidate.pe_ratio,
+            revenue_growth=candidate.revenue_growth,
+            earnings_surprise=candidate.earnings_surprise,
+            market_cap=candidate.market_cap,
+            next_earnings_date=candidate.next_earnings_date,
+            fetched_at=refreshed_at,
+            unavailable_fields_json=unavailable_json,
+        )
+        await db.execute(
+            stmt.on_conflict_do_update(
+                index_elements=["ticker_id"],
+                set_={
+                    "pe_ratio": stmt.excluded.pe_ratio,
+                    "revenue_growth": stmt.excluded.revenue_growth,
+                    "earnings_surprise": stmt.excluded.earnings_surprise,
+                    "market_cap": stmt.excluded.market_cap,
+                    "next_earnings_date": stmt.excluded.next_earnings_date,
+                    "fetched_at": stmt.excluded.fetched_at,
+                    "unavailable_fields_json": (
+                        stmt.excluded.unavailable_fields_json
+                    ),
+                },
+            )
+        )
+
+    dimension_ids = await _fundamental_dimension_ids(db, changed_ids)
+    composite_ids = await _composite_ids(db, changed_ids)
+    if dimension_ids:
+        await db.execute(
+            update(DimensionScore)
+            .where(DimensionScore.ticker_id.in_(dimension_ids))
+            .values(is_stale=True)
+        )
+    if composite_ids:
+        await db.execute(
+            update(CompositeScore)
+            .where(CompositeScore.ticker_id.in_(composite_ids))
+            .values(is_stale=True)
+        )
+
+    await db.commit()
+    return {
+        "enabled": True,
+        "refreshed": len(candidates),
+        "score_inputs_changed": len(changed_ids),
+        "dimension_scores_staled": len(dimension_ids),
+        "composite_scores_staled": len(composite_ids),
+    }
+
+
+async def _existing_by_ticker(
+    db: AsyncSession, ticker_ids: list[int]
+) -> dict[int, FundamentalData]:
+    if not ticker_ids:
+        return {}
+    rows = (
+        await db.execute(
+            select(FundamentalData).where(
+                FundamentalData.ticker_id.in_(ticker_ids)
+            )
+        )
+    ).scalars()
+    return {row.ticker_id: row for row in rows}
+
+
+async def _fundamental_dimension_ids(
+    db: AsyncSession, ticker_ids: set[int]
+) -> set[int]:
+    if not ticker_ids:
+        return set()
+    rows = await db.execute(
+        select(DimensionScore.ticker_id).where(
+            DimensionScore.ticker_id.in_(ticker_ids),
+            DimensionScore.dimension == "fundamental",
+        )
+    )
+    return set(rows.scalars())
+
+
+async def _composite_ids(
+    db: AsyncSession, ticker_ids: set[int]
+) -> set[int]:
+    if not ticker_ids:
+        return set()
+    rows = await db.execute(
+        select(CompositeScore.ticker_id).where(
+            CompositeScore.ticker_id.in_(ticker_ids)
+        )
+    )
+    return set(rows.scalars())
+
+
+def _score_inputs_changed(
+    existing: FundamentalData | None,
+    candidate: fundamentals_candidate_service.CandidateFundamentals,
+) -> bool:
+    if existing is None:
+        return True
+    return any(
+        getattr(existing, field) != getattr(candidate, field)
+        for field in _SCORE_FIELDS
+    )

app/services/fundamentals_candidate_service.py

@@ -0,0 +1,288 @@
+"""Local SEC/Dolt candidate values for the legacy fundamentals cache.
+
+This is the single read path shared by the A5 parity report and the activated
+``fundamental_data`` refresh.  It never contacts SEC or Dolt: every input comes
+from PostgreSQL, so price- and earnings-driven values can still refresh when an
+upstream import is unchanged or unavailable.
+"""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+from dataclasses import dataclass, field
+from datetime import date, datetime
+from typing import Any
+from zoneinfo import ZoneInfo
+
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.earnings_event import EarningsEvent
+from app.models.fundamental_snapshot import FundamentalSnapshot
+from app.models.ohlcv import OHLCVRecord
+from app.models.ticker import Ticker
+from app.services import fundamentals_derivation as deriv
+
+
+@dataclass(frozen=True)
+class CandidateFundamentals:
+    ticker_id: int
+    symbol: str
+    cik: str | None
+    pe_ratio: float | None
+    revenue_growth: float | None
+    earnings_surprise: float | None
+    market_cap: float | None
+    next_earnings_date: date | None
+    price_date: date | None
+    unavailable_fields: dict[str, str] = field(default_factory=dict)
+
+
+async def build_candidates(
+    db: AsyncSession,
+    *,
+    today: date | None = None,
+) -> list[CandidateFundamentals]:
+    """Derive current cache candidates using only already-stored data."""
+    today = today or datetime.now(ZoneInfo("America/New_York")).date()
+    tickers = list(
+        (await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
+    )
+    if not tickers:
+        return []
+
+    ticker_ids = [ticker.id for ticker in tickers]
+    ciks = sorted({ticker.cik for ticker in tickers if ticker.cik})
+    derived_by_cik = await _derived_by_cik(db, ciks)
+    closes_by_ticker = await _latest_closes(db, ticker_ids)
+    surprise_by_ticker, next_by_ticker = await _earnings_values(
+        db, ticker_ids, today
+    )
+
+    out: list[CandidateFundamentals] = []
+    for ticker in tickers:
+        derived = derived_by_cik.get(ticker.cik) if ticker.cik else None
+        close = closes_by_ticker.get(ticker.id)
+        price = close[0] if close is not None else None
+        price_date = close[1] if close is not None else None
+        growth_series = (
+            derived.metrics.get("revenue_growth_yoy")
+            if derived is not None
+            else None
+        )
+
+        pe_ratio = (
+            _pe(price, derived.ttm_diluted_eps)
+            if derived is not None
+            else None
+        )
+        revenue_growth = (
+            float(growth_series.value)
+            if growth_series is not None and _finite(growth_series.value)
+            else None
+        )
+        earnings_surprise = surprise_by_ticker.get(ticker.id)
+        market_cap = (
+            _market_cap(price, derived.shares_outstanding)
+            if derived is not None
+            else None
+        )
+        next_earnings_date = next_by_ticker.get(ticker.id)
+
+        out.append(
+            CandidateFundamentals(
+                ticker_id=ticker.id,
+                symbol=ticker.symbol,
+                cik=ticker.cik,
+                pe_ratio=pe_ratio,
+                revenue_growth=revenue_growth,
+                earnings_surprise=earnings_surprise,
+                market_cap=market_cap,
+                next_earnings_date=next_earnings_date,
+                price_date=price_date,
+                unavailable_fields=_availability_metadata(
+                    derived=derived,
+                    price=price,
+                    pe_ratio=pe_ratio,
+                    revenue_growth=revenue_growth,
+                    earnings_surprise=earnings_surprise,
+                    market_cap=market_cap,
+                    next_earnings_date=next_earnings_date,
+                ),
+            )
+        )
+    return out
+
+
+async def _derived_by_cik(
+    db: AsyncSession, ciks: list[str]
+) -> dict[str, deriv.DerivedFundamentals]:
+    if not ciks:
+        return {}
+    grouped: dict[str, list[FundamentalSnapshot]] = defaultdict(list)
+    rows = (
+        await db.execute(
+            select(FundamentalSnapshot).where(FundamentalSnapshot.cik.in_(ciks))
+        )
+    ).scalars()
+    for row in rows:
+        grouped[row.cik].append(row)
+    return {cik: deriv.derive(grouped.get(cik, [])) for cik in ciks}
+
+
+async def _latest_closes(
+    db: AsyncSession, ticker_ids: list[int]
+) -> dict[int, tuple[float, date]]:
+    latest = (
+        select(
+            OHLCVRecord.ticker_id,
+            func.max(OHLCVRecord.date).label("max_date"),
+        )
+        .where(OHLCVRecord.ticker_id.in_(ticker_ids))
+        .group_by(OHLCVRecord.ticker_id)
+        .subquery()
+    )
+    rows = (
+        await db.execute(
+            select(
+                OHLCVRecord.ticker_id,
+                OHLCVRecord.close,
+                OHLCVRecord.date,
+            ).join(
+                latest,
+                (OHLCVRecord.ticker_id == latest.c.ticker_id)
+                & (OHLCVRecord.date == latest.c.max_date),
+            )
+        )
+    ).all()
+    return {
+        ticker_id: (float(close), close_date)
+        for ticker_id, close, close_date in rows
+        if _finite(close)
+    }
+
+
+async def _earnings_values(
+    db: AsyncSession,
+    ticker_ids: list[int],
+    today: date,
+) -> tuple[dict[int, float], dict[int, date]]:
+    rows = (
+        await db.execute(
+            select(EarningsEvent)
+            .where(EarningsEvent.ticker_id.in_(ticker_ids))
+            .order_by(EarningsEvent.ticker_id, EarningsEvent.announce_date.desc())
+        )
+    ).scalars()
+    surprises: dict[int, float] = {}
+    upcoming: dict[int, date] = {}
+    for row in rows:
+        if row.announce_date >= today:
+            current = upcoming.get(row.ticker_id)
+            if current is None or row.announce_date < current:
+                upcoming[row.ticker_id] = row.announce_date
+            continue
+        if row.ticker_id in surprises:
+            continue
+        surprise = _surprise(row.eps_estimate, row.eps_actual)
+        if surprise is not None:
+            surprises[row.ticker_id] = surprise
+    return surprises, upcoming
+
+
+def _availability_metadata(
+    *,
+    derived: deriv.DerivedFundamentals | None,
+    price: float | None,
+    pe_ratio: float | None,
+    revenue_growth: float | None,
+    earnings_surprise: float | None,
+    market_cap: float | None,
+    next_earnings_date: date | None,
+) -> dict[str, str]:
+    metadata: dict[str, str] = {}
+
+    if pe_ratio is not None:
+        metadata["source_pe_ratio"] = "sec_facts+ohlcv_records"
+    elif derived is None or derived.latest_period_end is None:
+        metadata["pe_ratio"] = "no SEC fundamental snapshots"
+    elif not _finite(price) or price <= 0:
+        metadata["pe_ratio"] = "no usable PostgreSQL close"
+    elif derived.ttm_diluted_eps_caveat:
+        metadata["pe_ratio"] = derived.ttm_diluted_eps_caveat
+    else:
+        metadata["pe_ratio"] = "no positive SEC-derived TTM diluted EPS"
+
+    if revenue_growth is not None:
+        metadata["source_revenue_growth"] = "sec_facts"
+    else:
+        metadata["revenue_growth"] = "SEC-derived TTM revenue growth unavailable"
+
+    if earnings_surprise is not None:
+        metadata["source_earnings_surprise"] = "dolt_earnings"
+    else:
+        metadata["earnings_surprise"] = (
+            "no completed earnings event with actual and nonzero estimate"
+        )
+
+    if market_cap is not None:
+        metadata["source_market_cap"] = "sec_facts+ohlcv_records"
+        if derived is not None and derived.shares_outstanding_estimated:
+            metadata["market_cap_estimated"] = (
+                "shares use the SEC weighted-average diluted fallback"
+            )
+    elif derived is None or derived.latest_period_end is None:
+        metadata["market_cap"] = "no SEC fundamental snapshots"
+    elif not _finite(price) or price <= 0:
+        metadata["market_cap"] = "no usable PostgreSQL close"
+    else:
+        metadata["market_cap"] = "SEC-derived shares outstanding unavailable"
+
+    if next_earnings_date is not None:
+        metadata["source_next_earnings_date"] = "dolt_earnings"
+    else:
+        metadata["next_earnings_date"] = "no upcoming earnings event"
+    return metadata
+
+
+def _surprise(
+    estimate: float | None,
+    actual: float | None,
+) -> float | None:
+    if not _finite(estimate) or not _finite(actual) or estimate == 0:
+        return None
+    return (float(actual) - float(estimate)) / abs(float(estimate)) * 100.0
+
+
+def _pe(price: float | None, ttm_eps: float | None) -> float | None:
+    if (
+        not _finite(price)
+        or price <= 0
+        or not _finite(ttm_eps)
+        or ttm_eps <= 0
+    ):
+        return None
+    return float(price) / float(ttm_eps)
+
+
+def _market_cap(
+    price: float | None,
+    shares_outstanding: float | None,
+) -> float | None:
+    if (
+        not _finite(price)
+        or price <= 0
+        or not _finite(shares_outstanding)
+        or shares_outstanding <= 0
+    ):
+        return None
+    return float(price) * float(shares_outstanding)
+
+
+def _finite(value: Any) -> bool:
+    return (
+        isinstance(value, (int, float))
+        and not isinstance(value, bool)
+        and math.isfinite(value)
+    )

app/services/fundamentals_parity_service.py

@@ -14,22 +14,17 @@ import json
 import math
 import os
 import statistics
-from collections import defaultdict
 from datetime import date, datetime, timezone
 from pathlib import Path
 from typing import Any, Iterable
 from zoneinfo import ZoneInfo
 
-from sqlalchemy import func, select, text
+from sqlalchemy import select, text
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.models.data_import_run import DataImportRun
-from app.models.earnings_event import EarningsEvent
 from app.models.fundamental import FundamentalData
-from app.models.fundamental_snapshot import FundamentalSnapshot
-from app.models.ohlcv import OHLCVRecord
-from app.models.ticker import Ticker
-from app.services import fundamentals_derivation as deriv
+from app.services import fundamentals_candidate_service as candidate_service
 
 REPORT_VERSION = 1
 APPROVAL_STATUS = "pending_explicit_approval"
@@ -94,33 +89,18 @@ async def build_report(
         )
         await connection.execute(text("SET TRANSACTION READ ONLY"))
 
-    tickers = list((await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars())
-    ticker_ids = [ticker.id for ticker in tickers]
-    ciks = sorted({ticker.cik for ticker in tickers if ticker.cik})
-
+    candidates = await candidate_service.build_candidates(db, today=today)
+    ticker_ids = [candidate.ticker_id for candidate in candidates]
     legacy_by_ticker = await _legacy_values(db, ticker_ids)
-    derived_by_cik = await _derived_by_cik(db, ciks)
-    closes_by_ticker = await _latest_closes(db, ticker_ids)
-    surprise_by_ticker = await _latest_surprises(db, ticker_ids, today)
     source_runs = await _source_runs(db)
 
     rows: list[dict[str, Any]] = []
-    for ticker in tickers:
-        legacy = legacy_by_ticker.get(ticker.id)
-        derived = derived_by_cik.get(ticker.cik) if ticker.cik else None
-        close = closes_by_ticker.get(ticker.id)
-        candidate_pe = (
-            _pe(close[0], derived.ttm_diluted_eps)
-            if close is not None and derived is not None
-            else None
-        )
-        growth_series = (
-            derived.metrics.get("revenue_growth_yoy") if derived is not None else None
-        )
-        candidate = {
-            "pe_ratio": candidate_pe,
-            "revenue_growth": growth_series.value if growth_series else None,
-            "earnings_surprise": surprise_by_ticker.get(ticker.id),
+    for candidate in candidates:
+        legacy = legacy_by_ticker.get(candidate.ticker_id)
+        candidate_values = {
+            "pe_ratio": candidate.pe_ratio,
+            "revenue_growth": candidate.revenue_growth,
+            "earnings_surprise": candidate.earnings_surprise,
         }
         legacy_values = {
             "pe_ratio": legacy.pe_ratio if legacy else None,
@@ -128,17 +108,17 @@ async def build_report(
             "earnings_surprise": legacy.earnings_surprise if legacy else None,
         }
         fields = {
-            key: _field_comparison(key, legacy_values[key], candidate[key])
+            key: _field_comparison(key, legacy_values[key], candidate_values[key])
             for key in FIELD_KEYS
         }
         legacy_score = fundamental_score(**legacy_values)
-        candidate_score = fundamental_score(**candidate)
+        candidate_score = fundamental_score(**candidate_values)
         rows.append(
             {
-                "symbol": ticker.symbol,
-                "cik": ticker.cik,
+                "symbol": candidate.symbol,
+                "cik": candidate.cik,
                 "legacy_fetched_at": _iso(legacy.fetched_at) if legacy else None,
-                "price_date": _iso(close[1]) if close else None,
+                "price_date": _iso(candidate.price_date),
                 "fields": fields,
                 "scores": {
                     "legacy_fundamental": _round(legacy_score),
@@ -311,74 +291,6 @@ async def _legacy_values(
     return {row.ticker_id: row for row in rows}
 
 
-async def _derived_by_cik(
-    db: AsyncSession, ciks: list[str]
-) -> dict[str, deriv.DerivedFundamentals]:
-    if not ciks:
-        return {}
-    grouped: dict[str, list[FundamentalSnapshot]] = defaultdict(list)
-    rows = (
-        await db.execute(
-            select(FundamentalSnapshot).where(FundamentalSnapshot.cik.in_(ciks))
-        )
-    ).scalars()
-    for row in rows:
-        grouped[row.cik].append(row)
-    return {cik: deriv.derive(grouped.get(cik, [])) for cik in ciks}
-
-
-async def _latest_closes(
-    db: AsyncSession, ticker_ids: list[int]
-) -> dict[int, tuple[float, date]]:
-    if not ticker_ids:
-        return {}
-    latest = (
-        select(OHLCVRecord.ticker_id, func.max(OHLCVRecord.date).label("max_date"))
-        .where(OHLCVRecord.ticker_id.in_(ticker_ids))
-        .group_by(OHLCVRecord.ticker_id)
-        .subquery()
-    )
-    rows = (
-        await db.execute(
-            select(OHLCVRecord.ticker_id, OHLCVRecord.close, OHLCVRecord.date).join(
-                latest,
-                (OHLCVRecord.ticker_id == latest.c.ticker_id)
-                & (OHLCVRecord.date == latest.c.max_date),
-            )
-        )
-    ).all()
-    return {
-        ticker_id: (float(close), close_date)
-        for ticker_id, close, close_date in rows
-        if _finite(close)
-    }
-
-
-async def _latest_surprises(
-    db: AsyncSession, ticker_ids: list[int], today: date
-) -> dict[int, float]:
-    if not ticker_ids:
-        return {}
-    rows = (
-        await db.execute(
-            select(EarningsEvent)
-            .where(
-                EarningsEvent.ticker_id.in_(ticker_ids),
-                EarningsEvent.announce_date < today,
-            )
-            .order_by(EarningsEvent.ticker_id, EarningsEvent.announce_date.desc())
-        )
-    ).scalars()
-    out: dict[int, float] = {}
-    for row in rows:
-        if row.ticker_id in out:
-            continue
-        surprise = _surprise(row.eps_estimate, row.eps_actual)
-        if surprise is not None:
-            out[row.ticker_id] = surprise
-    return out
-
-
 async def _source_runs(db: AsyncSession) -> dict[str, dict[str, Any] | None]:
     sources = ("sec_facts", "dolt_earnings")
     rows = (
@@ -526,18 +438,6 @@ def _rank_change(legacy: int | None, candidate: int | None) -> int | None:
     return legacy - candidate if legacy is not None and candidate is not None else None
 
 
-def _surprise(estimate: float | None, actual: float | None) -> float | None:
-    if not _finite(estimate) or not _finite(actual) or estimate == 0:
-        return None
-    return (actual - estimate) / abs(estimate) * 100.0
-
-
-def _pe(price: float | None, ttm_eps: float | None) -> float | None:
-    if not _finite(price) or price <= 0 or not _finite(ttm_eps) or ttm_eps <= 0:
-        return None
-    return price / ttm_eps
-
-
 def _delta(legacy: float | None, candidate: float | None) -> float | None:
     if not _finite(legacy) or not _finite(candidate):
         return None