Implement A5 fundamentals cutover activation
This commit is contained in:
+77
-9
@@ -41,6 +41,7 @@ from app.services import (
|
|||||||
settings_store,
|
settings_store,
|
||||||
shadow_book_service,
|
shadow_book_service,
|
||||||
fundamentals_parity_service,
|
fundamentals_parity_service,
|
||||||
|
fundamental_data_refresh_service,
|
||||||
)
|
)
|
||||||
from app.services.data_import import STATUS_FAILED, SourceImporter, run_import
|
from app.services.data_import import STATUS_FAILED, SourceImporter, run_import
|
||||||
from app.services.dolt_earnings_importer import DoltEarningsImporter
|
from app.services.dolt_earnings_importer import DoltEarningsImporter
|
||||||
@@ -652,7 +653,7 @@ async def run_shadow_book() -> None:
|
|||||||
if not await _is_job_enabled(db, job_name):
|
if not await _is_job_enabled(db, job_name):
|
||||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
_log_event(logging.INFO, "job_skipped", job=job_name, reason="disabled")
|
||||||
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Disabled")
|
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Disabled")
|
||||||
return
|
return False
|
||||||
if not await shadow_book_service.is_enabled(db):
|
if not await shadow_book_service.is_enabled(db):
|
||||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="not enabled in settings")
|
_log_event(logging.INFO, "job_skipped", job=job_name, reason="not enabled in settings")
|
||||||
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Not enabled")
|
_runtime_finish(job_name, "skipped", processed=0, total=1, message="Not enabled")
|
||||||
@@ -924,8 +925,13 @@ async def collect_fundamentals() -> None:
|
|||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
async def _run_shadow_import(job_name: str, importer: SourceImporter) -> bool:
|
||||||
"""Run one source importer and surface its audit result in Admin → Jobs."""
|
"""Run an importer and return whether its scheduled job was enabled.
|
||||||
|
|
||||||
|
The SEC wrapper uses the return value to run its activated local cache step
|
||||||
|
after failed, no-op, promoted, or source-locked attempts while still honoring
|
||||||
|
the job-level disable switch.
|
||||||
|
"""
|
||||||
_log_event(logging.INFO, "job_start", job=job_name)
|
_log_event(logging.INFO, "job_start", job=job_name)
|
||||||
_runtime_start(job_name, total=1)
|
_runtime_start(job_name, total=1)
|
||||||
|
|
||||||
@@ -941,7 +947,7 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
|||||||
message = "Another import for this source is already running"
|
message = "Another import for this source is already running"
|
||||||
_log_event(logging.INFO, "job_skipped", job=job_name, reason="source_locked")
|
_log_event(logging.INFO, "job_skipped", job=job_name, reason="source_locked")
|
||||||
_runtime_finish(job_name, "skipped", processed=0, total=1, message=message)
|
_runtime_finish(job_name, "skipped", processed=0, total=1, message=message)
|
||||||
return
|
return True
|
||||||
|
|
||||||
revision = f" · {run.revision[:12]}" if run.revision else ""
|
revision = f" · {run.revision[:12]}" if run.revision else ""
|
||||||
message = f"{run.status}{revision}"
|
message = f"{run.status}{revision}"
|
||||||
@@ -949,7 +955,7 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
|||||||
message = run.error_details or message
|
message = run.error_details or message
|
||||||
_log_event(logging.ERROR, "job_error", job=job_name, message=message)
|
_log_event(logging.ERROR, "job_error", job=job_name, message=message)
|
||||||
_runtime_finish(job_name, "error", processed=0, total=1, message=message)
|
_runtime_finish(job_name, "error", processed=0, total=1, message=message)
|
||||||
return
|
return True
|
||||||
|
|
||||||
_log_event(
|
_log_event(
|
||||||
logging.INFO,
|
logging.INFO,
|
||||||
@@ -959,6 +965,7 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
|||||||
revision=run.revision,
|
revision=run.revision,
|
||||||
)
|
)
|
||||||
_runtime_finish(job_name, "completed", processed=1, total=1, message=message)
|
_runtime_finish(job_name, "completed", processed=1, total=1, message=message)
|
||||||
|
return True
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
_runtime_finish(job_name, "error", processed=0, total=1, message="Cancelled")
|
_runtime_finish(job_name, "error", processed=0, total=1, message="Cancelled")
|
||||||
raise
|
raise
|
||||||
@@ -971,6 +978,7 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> None:
|
|||||||
message=str(exc),
|
message=str(exc),
|
||||||
)
|
)
|
||||||
_runtime_finish(job_name, "error", processed=0, total=1, message=str(exc))
|
_runtime_finish(job_name, "error", processed=0, total=1, message=str(exc))
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
async def run_dolt_earnings_import() -> None:
|
async def run_dolt_earnings_import() -> None:
|
||||||
@@ -979,8 +987,67 @@ async def run_dolt_earnings_import() -> None:
|
|||||||
|
|
||||||
|
|
||||||
async def run_sec_fundamentals_import() -> None:
|
async def run_sec_fundamentals_import() -> None:
|
||||||
"""Import tracked-universe SEC facts in shadow."""
|
"""Import SEC facts, then run the activated local compat-cache refresh.
|
||||||
await _run_shadow_import("sec_fundamentals_import", SecFundamentalsImporter())
|
|
||||||
|
The refresh is deliberately separate from the network import result. Once
|
||||||
|
activated it therefore still runs from stored snapshots/earnings/prices when
|
||||||
|
SEC is unavailable, unchanged, or another SEC import owns the source lock.
|
||||||
|
"""
|
||||||
|
job_name = "sec_fundamentals_import"
|
||||||
|
job_enabled = await _run_shadow_import(job_name, SecFundamentalsImporter())
|
||||||
|
if not job_enabled:
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
async with async_session_factory() as db:
|
||||||
|
summary = await fundamental_data_refresh_service.refresh_if_enabled(db)
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
_runtime_finish(
|
||||||
|
job_name, "error", processed=0, total=1, message="Cancelled"
|
||||||
|
)
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
message = f"Local fundamental_data refresh failed: {exc}"
|
||||||
|
_log_event(
|
||||||
|
logging.ERROR,
|
||||||
|
"fundamental_data_refresh_error",
|
||||||
|
job=job_name,
|
||||||
|
error_type=type(exc).__name__,
|
||||||
|
message=str(exc),
|
||||||
|
)
|
||||||
|
_runtime_finish(job_name, "error", processed=0, total=1, message=message)
|
||||||
|
return
|
||||||
|
|
||||||
|
if not summary["enabled"]:
|
||||||
|
_log_event(
|
||||||
|
logging.INFO,
|
||||||
|
"fundamental_data_refresh_skipped",
|
||||||
|
job=job_name,
|
||||||
|
reason="cutover_disabled",
|
||||||
|
setting=fundamental_data_refresh_service.ACTIVATION_KEY,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
_log_event(
|
||||||
|
logging.INFO,
|
||||||
|
"fundamental_data_refresh_complete",
|
||||||
|
job=job_name,
|
||||||
|
**summary,
|
||||||
|
)
|
||||||
|
runtime = get_job_runtime_snapshot(job_name)
|
||||||
|
if runtime.get("status") == "completed":
|
||||||
|
import_message = runtime.get("message") or "import completed"
|
||||||
|
cache_message = (
|
||||||
|
f"cache {summary['refreshed']} · "
|
||||||
|
f"{summary['score_inputs_changed']} score inputs changed"
|
||||||
|
)
|
||||||
|
_runtime_finish(
|
||||||
|
job_name,
|
||||||
|
"completed",
|
||||||
|
processed=1,
|
||||||
|
total=1,
|
||||||
|
message=f"{import_message} · {cache_message}",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def run_fundamentals_parity_report() -> None:
|
async def run_fundamentals_parity_report() -> None:
|
||||||
@@ -1566,7 +1633,8 @@ SCHEDULE_DEFAULTS: dict[str, str] = {
|
|||||||
"schedule_timezone": "America/New_York",
|
"schedule_timezone": "America/New_York",
|
||||||
# Morning data/display refresh (no qualifying R:R scan).
|
# Morning data/display refresh (no qualifying R:R scan).
|
||||||
"schedule_daily_pipeline_cron": "0 2 * * *",
|
"schedule_daily_pipeline_cron": "0 2 * * *",
|
||||||
# Shadow source imports. They never write legacy fundamental_data before A5.
|
# Bulk source imports. The SEC job writes the legacy compat cache only after
|
||||||
|
# the explicit, default-off A5 cutover setting is enabled.
|
||||||
"schedule_dolt_earnings_cron": "30 2 * * *",
|
"schedule_dolt_earnings_cron": "30 2 * * *",
|
||||||
"schedule_sec_fundamentals_cron": "0 4 * * *",
|
"schedule_sec_fundamentals_cron": "0 4 * * *",
|
||||||
"schedule_fundamentals_parity_cron": "30 5 * * *",
|
"schedule_fundamentals_parity_cron": "30 5 * * *",
|
||||||
@@ -1689,7 +1757,7 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
|||||||
"schedule_sec_fundamentals_cron",
|
"schedule_sec_fundamentals_cron",
|
||||||
),
|
),
|
||||||
id="sec_fundamentals_import",
|
id="sec_fundamentals_import",
|
||||||
name="SEC Fundamentals Import (shadow)",
|
name="SEC Fundamentals Import",
|
||||||
replace_existing=True,
|
replace_existing=True,
|
||||||
)
|
)
|
||||||
scheduler.add_job(
|
scheduler.add_job(
|
||||||
|
|||||||
@@ -637,7 +637,7 @@ JOB_LABELS = {
|
|||||||
"sentiment_collector": "Sentiment Collector",
|
"sentiment_collector": "Sentiment Collector",
|
||||||
"fundamental_collector": "Fundamental Collector",
|
"fundamental_collector": "Fundamental Collector",
|
||||||
"dolt_earnings_import": "Dolt Earnings Import (shadow)",
|
"dolt_earnings_import": "Dolt Earnings Import (shadow)",
|
||||||
"sec_fundamentals_import": "SEC Fundamentals Import (shadow)",
|
"sec_fundamentals_import": "SEC Fundamentals Import",
|
||||||
"fundamentals_parity_report": "Fundamentals Parity Report (read-only)",
|
"fundamentals_parity_report": "Fundamentals Parity Report (read-only)",
|
||||||
"rr_scanner": "R:R Scanner",
|
"rr_scanner": "R:R Scanner",
|
||||||
"ticker_universe_sync": "Ticker Universe Sync",
|
"ticker_universe_sync": "Ticker Universe Sync",
|
||||||
|
|||||||
@@ -0,0 +1,179 @@
|
|||||||
|
"""A5 activation: refresh the legacy fundamentals cache from local bulk data."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from datetime import date, datetime, timezone
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from sqlalchemy import select, update
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.database import insert_for_session
|
||||||
|
from app.models.fundamental import FundamentalData
|
||||||
|
from app.models.score import CompositeScore, DimensionScore
|
||||||
|
from app.services import fundamentals_candidate_service, settings_store
|
||||||
|
|
||||||
|
|
||||||
|
# Absence is deliberately false. Production activation therefore requires one
|
||||||
|
# explicit, durable SystemSetting change after the A5 evidence is approved.
|
||||||
|
ACTIVATION_KEY = "fundamental_data_sec_dolt_cutover_enabled"
|
||||||
|
_SCORE_FIELDS = ("pe_ratio", "revenue_growth", "earnings_surprise")
|
||||||
|
|
||||||
|
|
||||||
|
async def is_enabled(db: AsyncSession) -> bool:
|
||||||
|
raw = await settings_store.get_value(db, ACTIVATION_KEY, "false")
|
||||||
|
return str(raw).strip().lower() == "true"
|
||||||
|
|
||||||
|
|
||||||
|
async def refresh_if_enabled(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
today: date | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Refresh atomically when activated; otherwise perform no writes."""
|
||||||
|
if not await is_enabled(db):
|
||||||
|
return {
|
||||||
|
"enabled": False,
|
||||||
|
"refreshed": 0,
|
||||||
|
"score_inputs_changed": 0,
|
||||||
|
"dimension_scores_staled": 0,
|
||||||
|
"composite_scores_staled": 0,
|
||||||
|
}
|
||||||
|
return await refresh(db, now=now, today=today)
|
||||||
|
|
||||||
|
|
||||||
|
async def refresh(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
now: datetime | None = None,
|
||||||
|
today: date | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Replace every ticker's compat-cache row in one database transaction.
|
||||||
|
|
||||||
|
Candidate values are assembled before the first write and use only local
|
||||||
|
PostgreSQL tables. A failure rolls the whole refresh back. Only changes to
|
||||||
|
the three scoring inputs invalidate cached scores; market cap and the next
|
||||||
|
earnings date are display-only.
|
||||||
|
"""
|
||||||
|
refreshed_at = now or datetime.now(timezone.utc)
|
||||||
|
candidates = await fundamentals_candidate_service.build_candidates(
|
||||||
|
db, today=today
|
||||||
|
)
|
||||||
|
ticker_ids = [candidate.ticker_id for candidate in candidates]
|
||||||
|
existing = await _existing_by_ticker(db, ticker_ids)
|
||||||
|
changed_ids = {
|
||||||
|
candidate.ticker_id
|
||||||
|
for candidate in candidates
|
||||||
|
if _score_inputs_changed(existing.get(candidate.ticker_id), candidate)
|
||||||
|
}
|
||||||
|
|
||||||
|
for candidate in candidates:
|
||||||
|
unavailable_json = json.dumps(
|
||||||
|
candidate.unavailable_fields, sort_keys=True
|
||||||
|
)
|
||||||
|
stmt = insert_for_session(db, FundamentalData).values(
|
||||||
|
ticker_id=candidate.ticker_id,
|
||||||
|
pe_ratio=candidate.pe_ratio,
|
||||||
|
revenue_growth=candidate.revenue_growth,
|
||||||
|
earnings_surprise=candidate.earnings_surprise,
|
||||||
|
market_cap=candidate.market_cap,
|
||||||
|
next_earnings_date=candidate.next_earnings_date,
|
||||||
|
fetched_at=refreshed_at,
|
||||||
|
unavailable_fields_json=unavailable_json,
|
||||||
|
)
|
||||||
|
await db.execute(
|
||||||
|
stmt.on_conflict_do_update(
|
||||||
|
index_elements=["ticker_id"],
|
||||||
|
set_={
|
||||||
|
"pe_ratio": stmt.excluded.pe_ratio,
|
||||||
|
"revenue_growth": stmt.excluded.revenue_growth,
|
||||||
|
"earnings_surprise": stmt.excluded.earnings_surprise,
|
||||||
|
"market_cap": stmt.excluded.market_cap,
|
||||||
|
"next_earnings_date": stmt.excluded.next_earnings_date,
|
||||||
|
"fetched_at": stmt.excluded.fetched_at,
|
||||||
|
"unavailable_fields_json": (
|
||||||
|
stmt.excluded.unavailable_fields_json
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
dimension_ids = await _fundamental_dimension_ids(db, changed_ids)
|
||||||
|
composite_ids = await _composite_ids(db, changed_ids)
|
||||||
|
if dimension_ids:
|
||||||
|
await db.execute(
|
||||||
|
update(DimensionScore)
|
||||||
|
.where(DimensionScore.ticker_id.in_(dimension_ids))
|
||||||
|
.values(is_stale=True)
|
||||||
|
)
|
||||||
|
if composite_ids:
|
||||||
|
await db.execute(
|
||||||
|
update(CompositeScore)
|
||||||
|
.where(CompositeScore.ticker_id.in_(composite_ids))
|
||||||
|
.values(is_stale=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
await db.commit()
|
||||||
|
return {
|
||||||
|
"enabled": True,
|
||||||
|
"refreshed": len(candidates),
|
||||||
|
"score_inputs_changed": len(changed_ids),
|
||||||
|
"dimension_scores_staled": len(dimension_ids),
|
||||||
|
"composite_scores_staled": len(composite_ids),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _existing_by_ticker(
|
||||||
|
db: AsyncSession, ticker_ids: list[int]
|
||||||
|
) -> dict[int, FundamentalData]:
|
||||||
|
if not ticker_ids:
|
||||||
|
return {}
|
||||||
|
rows = (
|
||||||
|
await db.execute(
|
||||||
|
select(FundamentalData).where(
|
||||||
|
FundamentalData.ticker_id.in_(ticker_ids)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).scalars()
|
||||||
|
return {row.ticker_id: row for row in rows}
|
||||||
|
|
||||||
|
|
||||||
|
async def _fundamental_dimension_ids(
|
||||||
|
db: AsyncSession, ticker_ids: set[int]
|
||||||
|
) -> set[int]:
|
||||||
|
if not ticker_ids:
|
||||||
|
return set()
|
||||||
|
rows = await db.execute(
|
||||||
|
select(DimensionScore.ticker_id).where(
|
||||||
|
DimensionScore.ticker_id.in_(ticker_ids),
|
||||||
|
DimensionScore.dimension == "fundamental",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return set(rows.scalars())
|
||||||
|
|
||||||
|
|
||||||
|
async def _composite_ids(
|
||||||
|
db: AsyncSession, ticker_ids: set[int]
|
||||||
|
) -> set[int]:
|
||||||
|
if not ticker_ids:
|
||||||
|
return set()
|
||||||
|
rows = await db.execute(
|
||||||
|
select(CompositeScore.ticker_id).where(
|
||||||
|
CompositeScore.ticker_id.in_(ticker_ids)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return set(rows.scalars())
|
||||||
|
|
||||||
|
|
||||||
|
def _score_inputs_changed(
|
||||||
|
existing: FundamentalData | None,
|
||||||
|
candidate: fundamentals_candidate_service.CandidateFundamentals,
|
||||||
|
) -> bool:
|
||||||
|
if existing is None:
|
||||||
|
return True
|
||||||
|
return any(
|
||||||
|
getattr(existing, field) != getattr(candidate, field)
|
||||||
|
for field in _SCORE_FIELDS
|
||||||
|
)
|
||||||
@@ -0,0 +1,288 @@
|
|||||||
|
"""Local SEC/Dolt candidate values for the legacy fundamentals cache.
|
||||||
|
|
||||||
|
This is the single read path shared by the A5 parity report and the activated
|
||||||
|
``fundamental_data`` refresh. It never contacts SEC or Dolt: every input comes
|
||||||
|
from PostgreSQL, so price- and earnings-driven values can still refresh when an
|
||||||
|
upstream import is unchanged or unavailable.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import math
|
||||||
|
from collections import defaultdict
|
||||||
|
from dataclasses import dataclass, field
|
||||||
|
from datetime import date, datetime
|
||||||
|
from typing import Any
|
||||||
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
|
from sqlalchemy import func, select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
|
from app.models.earnings_event import EarningsEvent
|
||||||
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||||
|
from app.models.ohlcv import OHLCVRecord
|
||||||
|
from app.models.ticker import Ticker
|
||||||
|
from app.services import fundamentals_derivation as deriv
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class CandidateFundamentals:
|
||||||
|
ticker_id: int
|
||||||
|
symbol: str
|
||||||
|
cik: str | None
|
||||||
|
pe_ratio: float | None
|
||||||
|
revenue_growth: float | None
|
||||||
|
earnings_surprise: float | None
|
||||||
|
market_cap: float | None
|
||||||
|
next_earnings_date: date | None
|
||||||
|
price_date: date | None
|
||||||
|
unavailable_fields: dict[str, str] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
|
async def build_candidates(
|
||||||
|
db: AsyncSession,
|
||||||
|
*,
|
||||||
|
today: date | None = None,
|
||||||
|
) -> list[CandidateFundamentals]:
|
||||||
|
"""Derive current cache candidates using only already-stored data."""
|
||||||
|
today = today or datetime.now(ZoneInfo("America/New_York")).date()
|
||||||
|
tickers = list(
|
||||||
|
(await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars()
|
||||||
|
)
|
||||||
|
if not tickers:
|
||||||
|
return []
|
||||||
|
|
||||||
|
ticker_ids = [ticker.id for ticker in tickers]
|
||||||
|
ciks = sorted({ticker.cik for ticker in tickers if ticker.cik})
|
||||||
|
derived_by_cik = await _derived_by_cik(db, ciks)
|
||||||
|
closes_by_ticker = await _latest_closes(db, ticker_ids)
|
||||||
|
surprise_by_ticker, next_by_ticker = await _earnings_values(
|
||||||
|
db, ticker_ids, today
|
||||||
|
)
|
||||||
|
|
||||||
|
out: list[CandidateFundamentals] = []
|
||||||
|
for ticker in tickers:
|
||||||
|
derived = derived_by_cik.get(ticker.cik) if ticker.cik else None
|
||||||
|
close = closes_by_ticker.get(ticker.id)
|
||||||
|
price = close[0] if close is not None else None
|
||||||
|
price_date = close[1] if close is not None else None
|
||||||
|
growth_series = (
|
||||||
|
derived.metrics.get("revenue_growth_yoy")
|
||||||
|
if derived is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
|
||||||
|
pe_ratio = (
|
||||||
|
_pe(price, derived.ttm_diluted_eps)
|
||||||
|
if derived is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
revenue_growth = (
|
||||||
|
float(growth_series.value)
|
||||||
|
if growth_series is not None and _finite(growth_series.value)
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
earnings_surprise = surprise_by_ticker.get(ticker.id)
|
||||||
|
market_cap = (
|
||||||
|
_market_cap(price, derived.shares_outstanding)
|
||||||
|
if derived is not None
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
next_earnings_date = next_by_ticker.get(ticker.id)
|
||||||
|
|
||||||
|
out.append(
|
||||||
|
CandidateFundamentals(
|
||||||
|
ticker_id=ticker.id,
|
||||||
|
symbol=ticker.symbol,
|
||||||
|
cik=ticker.cik,
|
||||||
|
pe_ratio=pe_ratio,
|
||||||
|
revenue_growth=revenue_growth,
|
||||||
|
earnings_surprise=earnings_surprise,
|
||||||
|
market_cap=market_cap,
|
||||||
|
next_earnings_date=next_earnings_date,
|
||||||
|
price_date=price_date,
|
||||||
|
unavailable_fields=_availability_metadata(
|
||||||
|
derived=derived,
|
||||||
|
price=price,
|
||||||
|
pe_ratio=pe_ratio,
|
||||||
|
revenue_growth=revenue_growth,
|
||||||
|
earnings_surprise=earnings_surprise,
|
||||||
|
market_cap=market_cap,
|
||||||
|
next_earnings_date=next_earnings_date,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
async def _derived_by_cik(
|
||||||
|
db: AsyncSession, ciks: list[str]
|
||||||
|
) -> dict[str, deriv.DerivedFundamentals]:
|
||||||
|
if not ciks:
|
||||||
|
return {}
|
||||||
|
grouped: dict[str, list[FundamentalSnapshot]] = defaultdict(list)
|
||||||
|
rows = (
|
||||||
|
await db.execute(
|
||||||
|
select(FundamentalSnapshot).where(FundamentalSnapshot.cik.in_(ciks))
|
||||||
|
)
|
||||||
|
).scalars()
|
||||||
|
for row in rows:
|
||||||
|
grouped[row.cik].append(row)
|
||||||
|
return {cik: deriv.derive(grouped.get(cik, [])) for cik in ciks}
|
||||||
|
|
||||||
|
|
||||||
|
async def _latest_closes(
|
||||||
|
db: AsyncSession, ticker_ids: list[int]
|
||||||
|
) -> dict[int, tuple[float, date]]:
|
||||||
|
latest = (
|
||||||
|
select(
|
||||||
|
OHLCVRecord.ticker_id,
|
||||||
|
func.max(OHLCVRecord.date).label("max_date"),
|
||||||
|
)
|
||||||
|
.where(OHLCVRecord.ticker_id.in_(ticker_ids))
|
||||||
|
.group_by(OHLCVRecord.ticker_id)
|
||||||
|
.subquery()
|
||||||
|
)
|
||||||
|
rows = (
|
||||||
|
await db.execute(
|
||||||
|
select(
|
||||||
|
OHLCVRecord.ticker_id,
|
||||||
|
OHLCVRecord.close,
|
||||||
|
OHLCVRecord.date,
|
||||||
|
).join(
|
||||||
|
latest,
|
||||||
|
(OHLCVRecord.ticker_id == latest.c.ticker_id)
|
||||||
|
& (OHLCVRecord.date == latest.c.max_date),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
).all()
|
||||||
|
return {
|
||||||
|
ticker_id: (float(close), close_date)
|
||||||
|
for ticker_id, close, close_date in rows
|
||||||
|
if _finite(close)
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
async def _earnings_values(
|
||||||
|
db: AsyncSession,
|
||||||
|
ticker_ids: list[int],
|
||||||
|
today: date,
|
||||||
|
) -> tuple[dict[int, float], dict[int, date]]:
|
||||||
|
rows = (
|
||||||
|
await db.execute(
|
||||||
|
select(EarningsEvent)
|
||||||
|
.where(EarningsEvent.ticker_id.in_(ticker_ids))
|
||||||
|
.order_by(EarningsEvent.ticker_id, EarningsEvent.announce_date.desc())
|
||||||
|
)
|
||||||
|
).scalars()
|
||||||
|
surprises: dict[int, float] = {}
|
||||||
|
upcoming: dict[int, date] = {}
|
||||||
|
for row in rows:
|
||||||
|
if row.announce_date >= today:
|
||||||
|
current = upcoming.get(row.ticker_id)
|
||||||
|
if current is None or row.announce_date < current:
|
||||||
|
upcoming[row.ticker_id] = row.announce_date
|
||||||
|
continue
|
||||||
|
if row.ticker_id in surprises:
|
||||||
|
continue
|
||||||
|
surprise = _surprise(row.eps_estimate, row.eps_actual)
|
||||||
|
if surprise is not None:
|
||||||
|
surprises[row.ticker_id] = surprise
|
||||||
|
return surprises, upcoming
|
||||||
|
|
||||||
|
|
||||||
|
def _availability_metadata(
|
||||||
|
*,
|
||||||
|
derived: deriv.DerivedFundamentals | None,
|
||||||
|
price: float | None,
|
||||||
|
pe_ratio: float | None,
|
||||||
|
revenue_growth: float | None,
|
||||||
|
earnings_surprise: float | None,
|
||||||
|
market_cap: float | None,
|
||||||
|
next_earnings_date: date | None,
|
||||||
|
) -> dict[str, str]:
|
||||||
|
metadata: dict[str, str] = {}
|
||||||
|
|
||||||
|
if pe_ratio is not None:
|
||||||
|
metadata["source_pe_ratio"] = "sec_facts+ohlcv_records"
|
||||||
|
elif derived is None or derived.latest_period_end is None:
|
||||||
|
metadata["pe_ratio"] = "no SEC fundamental snapshots"
|
||||||
|
elif not _finite(price) or price <= 0:
|
||||||
|
metadata["pe_ratio"] = "no usable PostgreSQL close"
|
||||||
|
elif derived.ttm_diluted_eps_caveat:
|
||||||
|
metadata["pe_ratio"] = derived.ttm_diluted_eps_caveat
|
||||||
|
else:
|
||||||
|
metadata["pe_ratio"] = "no positive SEC-derived TTM diluted EPS"
|
||||||
|
|
||||||
|
if revenue_growth is not None:
|
||||||
|
metadata["source_revenue_growth"] = "sec_facts"
|
||||||
|
else:
|
||||||
|
metadata["revenue_growth"] = "SEC-derived TTM revenue growth unavailable"
|
||||||
|
|
||||||
|
if earnings_surprise is not None:
|
||||||
|
metadata["source_earnings_surprise"] = "dolt_earnings"
|
||||||
|
else:
|
||||||
|
metadata["earnings_surprise"] = (
|
||||||
|
"no completed earnings event with actual and nonzero estimate"
|
||||||
|
)
|
||||||
|
|
||||||
|
if market_cap is not None:
|
||||||
|
metadata["source_market_cap"] = "sec_facts+ohlcv_records"
|
||||||
|
if derived is not None and derived.shares_outstanding_estimated:
|
||||||
|
metadata["market_cap_estimated"] = (
|
||||||
|
"shares use the SEC weighted-average diluted fallback"
|
||||||
|
)
|
||||||
|
elif derived is None or derived.latest_period_end is None:
|
||||||
|
metadata["market_cap"] = "no SEC fundamental snapshots"
|
||||||
|
elif not _finite(price) or price <= 0:
|
||||||
|
metadata["market_cap"] = "no usable PostgreSQL close"
|
||||||
|
else:
|
||||||
|
metadata["market_cap"] = "SEC-derived shares outstanding unavailable"
|
||||||
|
|
||||||
|
if next_earnings_date is not None:
|
||||||
|
metadata["source_next_earnings_date"] = "dolt_earnings"
|
||||||
|
else:
|
||||||
|
metadata["next_earnings_date"] = "no upcoming earnings event"
|
||||||
|
return metadata
|
||||||
|
|
||||||
|
|
||||||
|
def _surprise(
|
||||||
|
estimate: float | None,
|
||||||
|
actual: float | None,
|
||||||
|
) -> float | None:
|
||||||
|
if not _finite(estimate) or not _finite(actual) or estimate == 0:
|
||||||
|
return None
|
||||||
|
return (float(actual) - float(estimate)) / abs(float(estimate)) * 100.0
|
||||||
|
|
||||||
|
|
||||||
|
def _pe(price: float | None, ttm_eps: float | None) -> float | None:
|
||||||
|
if (
|
||||||
|
not _finite(price)
|
||||||
|
or price <= 0
|
||||||
|
or not _finite(ttm_eps)
|
||||||
|
or ttm_eps <= 0
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
return float(price) / float(ttm_eps)
|
||||||
|
|
||||||
|
|
||||||
|
def _market_cap(
|
||||||
|
price: float | None,
|
||||||
|
shares_outstanding: float | None,
|
||||||
|
) -> float | None:
|
||||||
|
if (
|
||||||
|
not _finite(price)
|
||||||
|
or price <= 0
|
||||||
|
or not _finite(shares_outstanding)
|
||||||
|
or shares_outstanding <= 0
|
||||||
|
):
|
||||||
|
return None
|
||||||
|
return float(price) * float(shares_outstanding)
|
||||||
|
|
||||||
|
|
||||||
|
def _finite(value: Any) -> bool:
|
||||||
|
return (
|
||||||
|
isinstance(value, (int, float))
|
||||||
|
and not isinstance(value, bool)
|
||||||
|
and math.isfinite(value)
|
||||||
|
)
|
||||||
@@ -14,22 +14,17 @@ import json
|
|||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
import statistics
|
import statistics
|
||||||
from collections import defaultdict
|
|
||||||
from datetime import date, datetime, timezone
|
from datetime import date, datetime, timezone
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
from zoneinfo import ZoneInfo
|
from zoneinfo import ZoneInfo
|
||||||
|
|
||||||
from sqlalchemy import func, select, text
|
from sqlalchemy import select, text
|
||||||
from sqlalchemy.ext.asyncio import AsyncSession
|
from sqlalchemy.ext.asyncio import AsyncSession
|
||||||
|
|
||||||
from app.models.data_import_run import DataImportRun
|
from app.models.data_import_run import DataImportRun
|
||||||
from app.models.earnings_event import EarningsEvent
|
|
||||||
from app.models.fundamental import FundamentalData
|
from app.models.fundamental import FundamentalData
|
||||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
from app.services import fundamentals_candidate_service as candidate_service
|
||||||
from app.models.ohlcv import OHLCVRecord
|
|
||||||
from app.models.ticker import Ticker
|
|
||||||
from app.services import fundamentals_derivation as deriv
|
|
||||||
|
|
||||||
REPORT_VERSION = 1
|
REPORT_VERSION = 1
|
||||||
APPROVAL_STATUS = "pending_explicit_approval"
|
APPROVAL_STATUS = "pending_explicit_approval"
|
||||||
@@ -94,33 +89,18 @@ async def build_report(
|
|||||||
)
|
)
|
||||||
await connection.execute(text("SET TRANSACTION READ ONLY"))
|
await connection.execute(text("SET TRANSACTION READ ONLY"))
|
||||||
|
|
||||||
tickers = list((await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars())
|
candidates = await candidate_service.build_candidates(db, today=today)
|
||||||
ticker_ids = [ticker.id for ticker in tickers]
|
ticker_ids = [candidate.ticker_id for candidate in candidates]
|
||||||
ciks = sorted({ticker.cik for ticker in tickers if ticker.cik})
|
|
||||||
|
|
||||||
legacy_by_ticker = await _legacy_values(db, ticker_ids)
|
legacy_by_ticker = await _legacy_values(db, ticker_ids)
|
||||||
derived_by_cik = await _derived_by_cik(db, ciks)
|
|
||||||
closes_by_ticker = await _latest_closes(db, ticker_ids)
|
|
||||||
surprise_by_ticker = await _latest_surprises(db, ticker_ids, today)
|
|
||||||
source_runs = await _source_runs(db)
|
source_runs = await _source_runs(db)
|
||||||
|
|
||||||
rows: list[dict[str, Any]] = []
|
rows: list[dict[str, Any]] = []
|
||||||
for ticker in tickers:
|
for candidate in candidates:
|
||||||
legacy = legacy_by_ticker.get(ticker.id)
|
legacy = legacy_by_ticker.get(candidate.ticker_id)
|
||||||
derived = derived_by_cik.get(ticker.cik) if ticker.cik else None
|
candidate_values = {
|
||||||
close = closes_by_ticker.get(ticker.id)
|
"pe_ratio": candidate.pe_ratio,
|
||||||
candidate_pe = (
|
"revenue_growth": candidate.revenue_growth,
|
||||||
_pe(close[0], derived.ttm_diluted_eps)
|
"earnings_surprise": candidate.earnings_surprise,
|
||||||
if close is not None and derived is not None
|
|
||||||
else None
|
|
||||||
)
|
|
||||||
growth_series = (
|
|
||||||
derived.metrics.get("revenue_growth_yoy") if derived is not None else None
|
|
||||||
)
|
|
||||||
candidate = {
|
|
||||||
"pe_ratio": candidate_pe,
|
|
||||||
"revenue_growth": growth_series.value if growth_series else None,
|
|
||||||
"earnings_surprise": surprise_by_ticker.get(ticker.id),
|
|
||||||
}
|
}
|
||||||
legacy_values = {
|
legacy_values = {
|
||||||
"pe_ratio": legacy.pe_ratio if legacy else None,
|
"pe_ratio": legacy.pe_ratio if legacy else None,
|
||||||
@@ -128,17 +108,17 @@ async def build_report(
|
|||||||
"earnings_surprise": legacy.earnings_surprise if legacy else None,
|
"earnings_surprise": legacy.earnings_surprise if legacy else None,
|
||||||
}
|
}
|
||||||
fields = {
|
fields = {
|
||||||
key: _field_comparison(key, legacy_values[key], candidate[key])
|
key: _field_comparison(key, legacy_values[key], candidate_values[key])
|
||||||
for key in FIELD_KEYS
|
for key in FIELD_KEYS
|
||||||
}
|
}
|
||||||
legacy_score = fundamental_score(**legacy_values)
|
legacy_score = fundamental_score(**legacy_values)
|
||||||
candidate_score = fundamental_score(**candidate)
|
candidate_score = fundamental_score(**candidate_values)
|
||||||
rows.append(
|
rows.append(
|
||||||
{
|
{
|
||||||
"symbol": ticker.symbol,
|
"symbol": candidate.symbol,
|
||||||
"cik": ticker.cik,
|
"cik": candidate.cik,
|
||||||
"legacy_fetched_at": _iso(legacy.fetched_at) if legacy else None,
|
"legacy_fetched_at": _iso(legacy.fetched_at) if legacy else None,
|
||||||
"price_date": _iso(close[1]) if close else None,
|
"price_date": _iso(candidate.price_date),
|
||||||
"fields": fields,
|
"fields": fields,
|
||||||
"scores": {
|
"scores": {
|
||||||
"legacy_fundamental": _round(legacy_score),
|
"legacy_fundamental": _round(legacy_score),
|
||||||
@@ -311,74 +291,6 @@ async def _legacy_values(
|
|||||||
return {row.ticker_id: row for row in rows}
|
return {row.ticker_id: row for row in rows}
|
||||||
|
|
||||||
|
|
||||||
async def _derived_by_cik(
|
|
||||||
db: AsyncSession, ciks: list[str]
|
|
||||||
) -> dict[str, deriv.DerivedFundamentals]:
|
|
||||||
if not ciks:
|
|
||||||
return {}
|
|
||||||
grouped: dict[str, list[FundamentalSnapshot]] = defaultdict(list)
|
|
||||||
rows = (
|
|
||||||
await db.execute(
|
|
||||||
select(FundamentalSnapshot).where(FundamentalSnapshot.cik.in_(ciks))
|
|
||||||
)
|
|
||||||
).scalars()
|
|
||||||
for row in rows:
|
|
||||||
grouped[row.cik].append(row)
|
|
||||||
return {cik: deriv.derive(grouped.get(cik, [])) for cik in ciks}
|
|
||||||
|
|
||||||
|
|
||||||
async def _latest_closes(
|
|
||||||
db: AsyncSession, ticker_ids: list[int]
|
|
||||||
) -> dict[int, tuple[float, date]]:
|
|
||||||
if not ticker_ids:
|
|
||||||
return {}
|
|
||||||
latest = (
|
|
||||||
select(OHLCVRecord.ticker_id, func.max(OHLCVRecord.date).label("max_date"))
|
|
||||||
.where(OHLCVRecord.ticker_id.in_(ticker_ids))
|
|
||||||
.group_by(OHLCVRecord.ticker_id)
|
|
||||||
.subquery()
|
|
||||||
)
|
|
||||||
rows = (
|
|
||||||
await db.execute(
|
|
||||||
select(OHLCVRecord.ticker_id, OHLCVRecord.close, OHLCVRecord.date).join(
|
|
||||||
latest,
|
|
||||||
(OHLCVRecord.ticker_id == latest.c.ticker_id)
|
|
||||||
& (OHLCVRecord.date == latest.c.max_date),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
).all()
|
|
||||||
return {
|
|
||||||
ticker_id: (float(close), close_date)
|
|
||||||
for ticker_id, close, close_date in rows
|
|
||||||
if _finite(close)
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
async def _latest_surprises(
|
|
||||||
db: AsyncSession, ticker_ids: list[int], today: date
|
|
||||||
) -> dict[int, float]:
|
|
||||||
if not ticker_ids:
|
|
||||||
return {}
|
|
||||||
rows = (
|
|
||||||
await db.execute(
|
|
||||||
select(EarningsEvent)
|
|
||||||
.where(
|
|
||||||
EarningsEvent.ticker_id.in_(ticker_ids),
|
|
||||||
EarningsEvent.announce_date < today,
|
|
||||||
)
|
|
||||||
.order_by(EarningsEvent.ticker_id, EarningsEvent.announce_date.desc())
|
|
||||||
)
|
|
||||||
).scalars()
|
|
||||||
out: dict[int, float] = {}
|
|
||||||
for row in rows:
|
|
||||||
if row.ticker_id in out:
|
|
||||||
continue
|
|
||||||
surprise = _surprise(row.eps_estimate, row.eps_actual)
|
|
||||||
if surprise is not None:
|
|
||||||
out[row.ticker_id] = surprise
|
|
||||||
return out
|
|
||||||
|
|
||||||
|
|
||||||
async def _source_runs(db: AsyncSession) -> dict[str, dict[str, Any] | None]:
|
async def _source_runs(db: AsyncSession) -> dict[str, dict[str, Any] | None]:
|
||||||
sources = ("sec_facts", "dolt_earnings")
|
sources = ("sec_facts", "dolt_earnings")
|
||||||
rows = (
|
rows = (
|
||||||
@@ -526,18 +438,6 @@ def _rank_change(legacy: int | None, candidate: int | None) -> int | None:
|
|||||||
return legacy - candidate if legacy is not None and candidate is not None else None
|
return legacy - candidate if legacy is not None and candidate is not None else None
|
||||||
|
|
||||||
|
|
||||||
def _surprise(estimate: float | None, actual: float | None) -> float | None:
|
|
||||||
if not _finite(estimate) or not _finite(actual) or estimate == 0:
|
|
||||||
return None
|
|
||||||
return (actual - estimate) / abs(estimate) * 100.0
|
|
||||||
|
|
||||||
|
|
||||||
def _pe(price: float | None, ttm_eps: float | None) -> float | None:
|
|
||||||
if not _finite(price) or price <= 0 or not _finite(ttm_eps) or ttm_eps <= 0:
|
|
||||||
return None
|
|
||||||
return price / ttm_eps
|
|
||||||
|
|
||||||
|
|
||||||
def _delta(legacy: float | None, candidate: float | None) -> float | None:
|
def _delta(legacy: float | None, candidate: float | None) -> float | None:
|
||||||
if not _finite(legacy) or not _finite(candidate):
|
if not _finite(legacy) or not _finite(candidate):
|
||||||
return None
|
return None
|
||||||
|
|||||||
@@ -427,8 +427,9 @@ workstream B — Alpaca remains the price source throughout.
|
|||||||
fundamental-score/ranking changes, require explicit approval. Definition
|
fundamental-score/ranking changes, require explicit approval. Definition
|
||||||
changes (e.g. TTM vs provider convention) called out, not averaged away.
|
changes (e.g. TTM vs provider convention) called out, not averaged away.
|
||||||
**Status 2026-07-24: the gate has been exercised and the evidence supports
|
**Status 2026-07-24: the gate has been exercised and the evidence supports
|
||||||
approval** — see the handoff section below. What remains of A5 is the
|
approval** — see the handoff section below. Step (c) is implemented behind the
|
||||||
activation itself: implementing step (c) and flipping it on.
|
default-off `fundamental_data_sec_dolt_cutover_enabled` SystemSetting; the
|
||||||
|
remaining production action is flipping that switch on and observing it.
|
||||||
- A6. Remove FMP/Finnhub/Alpha Vantage; keep monitoring + manual fallback.
|
- A6. Remove FMP/Finnhub/Alpha Vantage; keep monitoring + manual fallback.
|
||||||
|
|
||||||
**Workstream B (independent, start when wanted):**
|
**Workstream B (independent, start when wanted):**
|
||||||
@@ -491,16 +492,18 @@ Post-fix: candidate scores 504 of 511 vs legacy's 507 (gap = PSKY/Q new registra
|
|||||||
FITB, all explained); revenue-growth agreement 0.0038 median abs delta where both exist.
|
FITB, all explained); revenue-growth agreement 0.0038 median abs delta where both exist.
|
||||||
Dennis reviewed the evidence 2026-07-24 and directed proceeding to cutover.
|
Dennis reviewed the evidence 2026-07-24 and directed proceeding to cutover.
|
||||||
|
|
||||||
**Task 1 — A5 activation (implement step (c) above, ~line 207).** The post-activation
|
**Task 1 — A5 activation (IMPLEMENTED 2026-07-24; production switch remains).** The
|
||||||
local refresh of `fundamental_data` does not exist yet. Per the spec: `pe_ratio` and
|
post-activation local refresh of `fundamental_data` derives `pe_ratio` and
|
||||||
`market_cap` from newest valid snapshots × latest PostgreSQL close, `revenue_growth`
|
`market_cap` from newest valid snapshots × latest PostgreSQL close, `revenue_growth`
|
||||||
from snapshots, `earnings_surprise`/`next_earnings_date` from `earnings_events`; mark
|
from snapshots, `earnings_surprise`/`next_earnings_date` from `earnings_events`; mark
|
||||||
affected cached fundamental scores stale; must run identically when SEC is unreachable.
|
affected cached fundamental scores stale; must run identically when SEC is unreachable.
|
||||||
Implementation notes from the parity work: consume `fundamentals_derivation.derive()`
|
It consumes `fundamentals_derivation.derive()` outputs, NOT raw snapshot fields —
|
||||||
outputs, NOT raw snapshot fields — that path carries the split guard (`ttm_diluted_eps`
|
that path carries the split guard (`ttm_diluted_eps`
|
||||||
nulls when contaminated, with `ttm_diluted_eps_caveat`) and the multi-class share
|
nulls when contaminated, with `ttm_diluted_eps_caveat`) and the multi-class share
|
||||||
fallback (`shares_outstanding` + `shares_outstanding_estimated`). Activation should be
|
fallback (`shares_outstanding` + `shares_outstanding_estimated`). Parity and activation
|
||||||
an explicit switch (SystemSetting, like `sec_cik_overrides`), default off.
|
share the same candidate builder. Activation is the explicit
|
||||||
|
`fundamental_data_sec_dolt_cutover_enabled` SystemSetting and defaults off; see
|
||||||
|
`docs/fundamentals-deployment.md` for the production flip and rollback procedure.
|
||||||
|
|
||||||
**Task 2 — A6 decommissioning.** After a short observation window: remove
|
**Task 2 — A6 decommissioning.** After a short observation window: remove
|
||||||
FMP/Finnhub/Alpha Vantage providers, config and env keys; keep monitoring + manual
|
FMP/Finnhub/Alpha Vantage providers, config and env keys; keep monitoring + manual
|
||||||
|
|||||||
@@ -1,13 +1,15 @@
|
|||||||
# Fundamentals production deployment
|
# Fundamentals production deployment
|
||||||
|
|
||||||
This is the one-time production setup for the Dolt earnings and SEC fundamentals
|
This is the one-time production setup for the Dolt earnings and SEC fundamentals
|
||||||
imports. Both imports remain shadow inputs until the separate A5 scoring-cutover
|
imports. The A5 scoring cutover was approved on 2026-07-24; the compat-cache write
|
||||||
approval. Do not add OS cron entries: the application scheduler owns both jobs.
|
path is still default-off until the explicit production switch below is set. Do
|
||||||
|
not add OS cron entries: the application scheduler owns both jobs.
|
||||||
|
|
||||||
## What the deployment adds
|
## What the deployment adds
|
||||||
|
|
||||||
- `Dolt Earnings Import (shadow)` runs daily at 02:30 America/New_York.
|
- `Dolt Earnings Import (shadow)` runs daily at 02:30 America/New_York.
|
||||||
- `SEC Fundamentals Import (shadow)` runs daily at 04:00 America/New_York.
|
- `SEC Fundamentals Import` runs daily at 04:00 America/New_York. Its local
|
||||||
|
`fundamental_data` refresh runs only when the A5 switch is enabled.
|
||||||
- `Fundamentals Parity Report (read-only)` runs daily at 05:30 America/New_York.
|
- `Fundamentals Parity Report (read-only)` runs daily at 05:30 America/New_York.
|
||||||
- Both jobs are visible, toggleable, and manually triggerable in Admin → Jobs.
|
- Both jobs are visible, toggleable, and manually triggerable in Admin → Jobs.
|
||||||
- Cron expressions are editable in Admin → Schedule.
|
- Cron expressions are editable in Admin → Schedule.
|
||||||
@@ -75,7 +77,7 @@ In Admin → Jobs, wait until no other job is running, then:
|
|||||||
|
|
||||||
1. Trigger **Dolt Earnings Import (shadow)**. Expect `completed` with import
|
1. Trigger **Dolt Earnings Import (shadow)**. Expect `completed` with import
|
||||||
status `promoted`; a repeat without an upstream change should report `no_op`.
|
status `promoted`; a repeat without an upstream change should report `no_op`.
|
||||||
2. Trigger **SEC Fundamentals Import (shadow)**. The first run performs the
|
2. Trigger **SEC Fundamentals Import**. The first run performs the
|
||||||
tracked-universe history backfill and can take materially longer than a daily
|
tracked-universe history backfill and can take materially longer than a daily
|
||||||
incremental run. Expect `completed` with import status `promoted`.
|
incremental run. Expect `completed` with import status `promoted`.
|
||||||
3. Check Admin → System Events. There should be no new import error.
|
3. Check Admin → System Events. There should be no new import error.
|
||||||
@@ -154,10 +156,69 @@ Expect `OK: source lock is busy`. This is the remaining live-PostgreSQL
|
|||||||
mutual-exclusion check; SQLite unit tests cannot exercise PostgreSQL advisory
|
mutual-exclusion check; SQLite unit tests cannot exercise PostgreSQL advisory
|
||||||
locks. A second Admin trigger should independently report the job as busy.
|
locks. A second Admin trigger should independently report the job as busy.
|
||||||
|
|
||||||
|
## A5 production activation (approved 2026-07-24)
|
||||||
|
|
||||||
|
The write path is controlled by the SystemSetting
|
||||||
|
`fundamental_data_sec_dolt_cutover_enabled`. An absent value, `false`, or any
|
||||||
|
value other than `true` leaves `fundamental_data` untouched. Before enabling it,
|
||||||
|
confirm the normal PostgreSQL backup containing `fundamental_data` is current.
|
||||||
|
|
||||||
|
Enable the cutover in PostgreSQL:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
INSERT INTO system_settings (key, value, updated_at)
|
||||||
|
VALUES ('fundamental_data_sec_dolt_cutover_enabled', 'true', now())
|
||||||
|
ON CONFLICT (key) DO UPDATE
|
||||||
|
SET value = EXCLUDED.value, updated_at = now();
|
||||||
|
```
|
||||||
|
|
||||||
|
Then trigger **SEC Fundamentals Import** once in Admin → Jobs. The import may be
|
||||||
|
`promoted` or `no_op`; either result runs the local refresh. Once enabled, the
|
||||||
|
same refresh also runs after an SEC network/validation failure or a source-lock
|
||||||
|
skip, because it reads only PostgreSQL snapshots, earnings events, and closes.
|
||||||
|
The job message appends the cache row count and changed score-input count when
|
||||||
|
the import itself completed successfully.
|
||||||
|
|
||||||
|
Verify the switch and refreshed rows:
|
||||||
|
|
||||||
|
```sql
|
||||||
|
SELECT key, value, updated_at
|
||||||
|
FROM system_settings
|
||||||
|
WHERE key = 'fundamental_data_sec_dolt_cutover_enabled';
|
||||||
|
|
||||||
|
SELECT count(*) AS rows,
|
||||||
|
max(fetched_at) AS refreshed_at,
|
||||||
|
count(pe_ratio) AS pe_available,
|
||||||
|
count(revenue_growth) AS growth_available,
|
||||||
|
count(earnings_surprise) AS surprise_available,
|
||||||
|
count(next_earnings_date) AS next_date_available
|
||||||
|
FROM fundamental_data;
|
||||||
|
|
||||||
|
SELECT dimension, is_stale, count(*)
|
||||||
|
FROM dimension_scores
|
||||||
|
WHERE dimension = 'fundamental'
|
||||||
|
GROUP BY dimension, is_stale;
|
||||||
|
|
||||||
|
SELECT is_stale, count(*)
|
||||||
|
FROM composite_scores
|
||||||
|
GROUP BY is_stale;
|
||||||
|
```
|
||||||
|
|
||||||
|
The first refresh intentionally marks affected fundamental and composite score
|
||||||
|
caches stale. The normal 15:30 near-close scanner recomputes them before using
|
||||||
|
the rankings; until then, reads truthfully expose the stale state. Observe at
|
||||||
|
least several scheduled cycles before A6 removes the legacy providers.
|
||||||
|
|
||||||
## Failure and rollback
|
## Failure and rollback
|
||||||
|
|
||||||
- Disable the failing shadow job in Admin → Jobs. This stops scheduled imports
|
- To stop the A5 cache writes without stopping SEC snapshot ingestion, set
|
||||||
without changing existing data or the legacy scoring path.
|
`fundamental_data_sec_dolt_cutover_enabled` back to `false` with the SQL above
|
||||||
|
(changing only the value). This prevents the next local refresh but does not
|
||||||
|
restore rows already replaced. Restore `fundamental_data` from the pre-cutover
|
||||||
|
database backup, or—before A6—manually run the legacy Fundamental Collector if
|
||||||
|
its provider keys and quota are still available.
|
||||||
|
- Disable a failing source-import job in Admin → Jobs only when ingestion itself
|
||||||
|
must stop. Existing promoted snapshots/events remain available.
|
||||||
- Inspect the job runtime, latest `data_import_runs.validation_json`, service
|
- Inspect the job runtime, latest `data_import_runs.validation_json`, service
|
||||||
logs, and Admin → System Events before retrying.
|
logs, and Admin → System Events before retrying.
|
||||||
- Re-run `sudo -u deploy bash ./deploy/provision_fundamentals.sh --check` for
|
- Re-run `sudo -u deploy bash ./deploy/provision_fundamentals.sh --check` for
|
||||||
@@ -165,8 +226,8 @@ locks. A second Admin trigger should independently report the job as busy.
|
|||||||
- The Dolt clone is a reproducible cache and does not need a bespoke backup.
|
- The Dolt clone is a reproducible cache and does not need a bespoke backup.
|
||||||
PostgreSQL (including `earnings_events`, `fundamental_snapshots`, and import
|
PostgreSQL (including `earnings_events`, `fundamental_snapshots`, and import
|
||||||
audit rows) must remain covered by the normal production database backup.
|
audit rows) must remain covered by the normal production database backup.
|
||||||
- Do not proceed to A5 while either shadow feed is unhealthy or the parity gate
|
- Do not proceed to A6 until the activated cache has completed the observation
|
||||||
has not received explicit approval.
|
window and the forward earnings calendar remains timely.
|
||||||
- If report generation fails, inspect Admin → System Events and verify
|
- If report generation fails, inspect Admin → System Events and verify
|
||||||
`FUNDAMENTALS_PARITY_REPORT_DIR` exists and is writable by `deploy`. Existing
|
`FUNDAMENTALS_PARITY_REPORT_DIR` exists and is writable by `deploy`. Existing
|
||||||
reports and all live data remain untouched.
|
reports and all live data remain untouched.
|
||||||
|
|||||||
@@ -0,0 +1,293 @@
|
|||||||
|
"""A5 activation: local candidate derivation and compat-cache refresh."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from datetime import date, datetime, timedelta, timezone
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import select
|
||||||
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||||
|
|
||||||
|
from app.database import Base
|
||||||
|
from app.models.earnings_event import EarningsEvent
|
||||||
|
from app.models.fundamental import FundamentalData
|
||||||
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||||
|
from app.models.ohlcv import OHLCVRecord
|
||||||
|
from app.models.score import CompositeScore, DimensionScore
|
||||||
|
from app.models.settings import SystemSetting
|
||||||
|
from app.models.ticker import Ticker
|
||||||
|
from app.services import fundamentals_candidate_service as candidates
|
||||||
|
from app.services import fundamentals_derivation as deriv
|
||||||
|
from app.services import fundamental_data_refresh_service as refresh_service
|
||||||
|
|
||||||
|
|
||||||
|
UTC = timezone.utc
|
||||||
|
NOW = datetime(2026, 7, 24, 10, 0, tzinfo=UTC)
|
||||||
|
TODAY = date(2026, 7, 24)
|
||||||
|
|
||||||
|
_engine = create_async_engine("sqlite+aiosqlite://", echo=False)
|
||||||
|
_session_factory = async_sessionmaker(
|
||||||
|
_engine, class_=AsyncSession, expire_on_commit=False
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
async def _setup_tables():
|
||||||
|
async with _engine.begin() as connection:
|
||||||
|
await connection.run_sync(Base.metadata.create_all)
|
||||||
|
yield
|
||||||
|
async with _engine.begin() as connection:
|
||||||
|
await connection.run_sync(Base.metadata.drop_all)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
async def session() -> AsyncSession:
|
||||||
|
async with _session_factory() as db:
|
||||||
|
yield db
|
||||||
|
|
||||||
|
|
||||||
|
def _snapshot_rows(cik: str) -> list[FundamentalSnapshot]:
|
||||||
|
rows: list[FundamentalSnapshot] = []
|
||||||
|
periods = ("Q1", "Q2", "Q3", "FY")
|
||||||
|
months = (3, 6, 9, 12)
|
||||||
|
for fiscal_year, multiplier in ((2025, 1.0), (2026, 1.1)):
|
||||||
|
revenue = [100 * multiplier, 110 * multiplier, 120 * multiplier, 130 * multiplier]
|
||||||
|
eps = [1.0 * multiplier, 1.1 * multiplier, 1.2 * multiplier, 1.3 * multiplier]
|
||||||
|
for index, fiscal_period in enumerate(periods):
|
||||||
|
period_end = date(fiscal_year, months[index], 28)
|
||||||
|
rows.append(
|
||||||
|
FundamentalSnapshot(
|
||||||
|
cik=cik,
|
||||||
|
accession=f"{cik}-{fiscal_year}-{fiscal_period}",
|
||||||
|
form="10-K" if fiscal_period == "FY" else "10-Q",
|
||||||
|
filed_date=period_end,
|
||||||
|
accepted_at=datetime(
|
||||||
|
fiscal_year, months[index], 28, tzinfo=UTC
|
||||||
|
),
|
||||||
|
period_end=period_end,
|
||||||
|
fiscal_year=fiscal_year,
|
||||||
|
fiscal_period=fiscal_period,
|
||||||
|
revenue=sum(revenue[: index + 1]),
|
||||||
|
diluted_eps=sum(eps[: index + 1]),
|
||||||
|
shares_outstanding=1_000,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
async def test_default_off_performs_no_candidate_read_or_write(
|
||||||
|
session: AsyncSession, monkeypatch
|
||||||
|
):
|
||||||
|
ticker = Ticker(symbol="AAA")
|
||||||
|
session.add(ticker)
|
||||||
|
await session.flush()
|
||||||
|
session.add(
|
||||||
|
FundamentalData(
|
||||||
|
ticker_id=ticker.id,
|
||||||
|
pe_ratio=12,
|
||||||
|
revenue_growth=3,
|
||||||
|
earnings_surprise=1,
|
||||||
|
market_cap=100,
|
||||||
|
fetched_at=NOW,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
async def should_not_read(*args, **kwargs):
|
||||||
|
raise AssertionError("default-off refresh derived candidates")
|
||||||
|
|
||||||
|
monkeypatch.setattr(candidates, "build_candidates", should_not_read)
|
||||||
|
summary = await refresh_service.refresh_if_enabled(session, today=TODAY)
|
||||||
|
|
||||||
|
stored = await session.scalar(
|
||||||
|
select(FundamentalData).where(FundamentalData.ticker_id == ticker.id)
|
||||||
|
)
|
||||||
|
assert summary == {
|
||||||
|
"enabled": False,
|
||||||
|
"refreshed": 0,
|
||||||
|
"score_inputs_changed": 0,
|
||||||
|
"dimension_scores_staled": 0,
|
||||||
|
"composite_scores_staled": 0,
|
||||||
|
}
|
||||||
|
assert stored.pe_ratio == 12
|
||||||
|
|
||||||
|
|
||||||
|
async def test_activated_refresh_updates_all_fields_and_invalidates_scores(
|
||||||
|
session: AsyncSession,
|
||||||
|
):
|
||||||
|
session.add(
|
||||||
|
SystemSetting(key=refresh_service.ACTIVATION_KEY, value="true")
|
||||||
|
)
|
||||||
|
first = Ticker(symbol="AAA", cik="0000000001")
|
||||||
|
second = Ticker(symbol="AAB", cik="0000000001")
|
||||||
|
session.add_all([first, second])
|
||||||
|
await session.flush()
|
||||||
|
session.add_all(_snapshot_rows(first.cik))
|
||||||
|
session.add_all(
|
||||||
|
[
|
||||||
|
OHLCVRecord(
|
||||||
|
ticker_id=first.id,
|
||||||
|
date=TODAY - timedelta(days=1),
|
||||||
|
open=100,
|
||||||
|
high=100,
|
||||||
|
low=100,
|
||||||
|
close=100,
|
||||||
|
volume=100,
|
||||||
|
),
|
||||||
|
OHLCVRecord(
|
||||||
|
ticker_id=second.id,
|
||||||
|
date=TODAY - timedelta(days=1),
|
||||||
|
open=200,
|
||||||
|
high=200,
|
||||||
|
low=200,
|
||||||
|
close=200,
|
||||||
|
volume=100,
|
||||||
|
),
|
||||||
|
EarningsEvent(
|
||||||
|
ticker_id=first.id,
|
||||||
|
announce_date=TODAY - timedelta(days=10),
|
||||||
|
session="amc",
|
||||||
|
eps_estimate=2,
|
||||||
|
eps_actual=2.2,
|
||||||
|
source="dolt_earnings",
|
||||||
|
),
|
||||||
|
EarningsEvent(
|
||||||
|
ticker_id=first.id,
|
||||||
|
announce_date=TODAY,
|
||||||
|
session="amc",
|
||||||
|
source="dolt_earnings",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
for ticker in (first, second):
|
||||||
|
session.add(
|
||||||
|
FundamentalData(
|
||||||
|
ticker_id=ticker.id,
|
||||||
|
pe_ratio=1,
|
||||||
|
revenue_growth=1,
|
||||||
|
earnings_surprise=1,
|
||||||
|
market_cap=1,
|
||||||
|
fetched_at=NOW - timedelta(days=1),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.add(
|
||||||
|
DimensionScore(
|
||||||
|
ticker_id=ticker.id,
|
||||||
|
dimension="fundamental",
|
||||||
|
score=50,
|
||||||
|
is_stale=False,
|
||||||
|
computed_at=NOW,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.add(
|
||||||
|
CompositeScore(
|
||||||
|
ticker_id=ticker.id,
|
||||||
|
score=50,
|
||||||
|
is_stale=False,
|
||||||
|
weights_json="{}",
|
||||||
|
computed_at=NOW,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
summary = await refresh_service.refresh_if_enabled(
|
||||||
|
session, now=NOW, today=TODAY
|
||||||
|
)
|
||||||
|
|
||||||
|
stored = {
|
||||||
|
row.ticker_id: row
|
||||||
|
for row in (
|
||||||
|
await session.execute(select(FundamentalData))
|
||||||
|
).scalars()
|
||||||
|
}
|
||||||
|
assert summary["refreshed"] == 2
|
||||||
|
assert summary["score_inputs_changed"] == 2
|
||||||
|
assert stored[first.id].pe_ratio == pytest.approx(100 / 5.06)
|
||||||
|
assert stored[second.id].pe_ratio == pytest.approx(200 / 5.06)
|
||||||
|
assert stored[first.id].revenue_growth == pytest.approx(10)
|
||||||
|
assert stored[first.id].earnings_surprise == pytest.approx(10)
|
||||||
|
assert stored[first.id].market_cap == pytest.approx(100_000)
|
||||||
|
assert stored[first.id].next_earnings_date == TODAY
|
||||||
|
metadata = json.loads(stored[first.id].unavailable_fields_json)
|
||||||
|
assert metadata["source_pe_ratio"] == "sec_facts+ohlcv_records"
|
||||||
|
assert metadata["source_next_earnings_date"] == "dolt_earnings"
|
||||||
|
|
||||||
|
dimensions = (
|
||||||
|
await session.execute(select(DimensionScore))
|
||||||
|
).scalars().all()
|
||||||
|
composites = (
|
||||||
|
await session.execute(select(CompositeScore))
|
||||||
|
).scalars().all()
|
||||||
|
assert all(row.is_stale for row in dimensions)
|
||||||
|
assert all(row.is_stale for row in composites)
|
||||||
|
|
||||||
|
for row in (*dimensions, *composites):
|
||||||
|
row.is_stale = False
|
||||||
|
await session.commit()
|
||||||
|
unchanged = await refresh_service.refresh_if_enabled(
|
||||||
|
session, now=NOW + timedelta(hours=1), today=TODAY
|
||||||
|
)
|
||||||
|
assert unchanged["score_inputs_changed"] == 0
|
||||||
|
assert not any(
|
||||||
|
(await session.execute(select(DimensionScore.is_stale))).scalars()
|
||||||
|
)
|
||||||
|
assert not any(
|
||||||
|
(await session.execute(select(CompositeScore.is_stale))).scalars()
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def test_candidate_uses_guarded_derive_outputs_and_share_fallback(
|
||||||
|
session: AsyncSession, monkeypatch
|
||||||
|
):
|
||||||
|
ticker = Ticker(symbol="GUARD", cik="0000000002")
|
||||||
|
session.add(ticker)
|
||||||
|
await session.flush()
|
||||||
|
session.add(
|
||||||
|
FundamentalSnapshot(
|
||||||
|
cik=ticker.cik,
|
||||||
|
accession="raw-accession",
|
||||||
|
form="10-Q",
|
||||||
|
filed_date=TODAY,
|
||||||
|
accepted_at=NOW,
|
||||||
|
period_end=TODAY,
|
||||||
|
fiscal_year=2026,
|
||||||
|
fiscal_period="Q2",
|
||||||
|
diluted_eps=99,
|
||||||
|
shares_outstanding=999,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
session.add(
|
||||||
|
OHLCVRecord(
|
||||||
|
ticker_id=ticker.id,
|
||||||
|
date=TODAY,
|
||||||
|
open=50,
|
||||||
|
high=50,
|
||||||
|
low=50,
|
||||||
|
close=50,
|
||||||
|
volume=100,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
await session.commit()
|
||||||
|
|
||||||
|
def guarded(_rows):
|
||||||
|
return deriv.DerivedFundamentals(
|
||||||
|
metrics={
|
||||||
|
"revenue_growth_yoy": deriv.MetricSeries(value=7)
|
||||||
|
},
|
||||||
|
ttm_diluted_eps=None,
|
||||||
|
ttm_diluted_eps_caveat="split guard applied",
|
||||||
|
shares_outstanding=123,
|
||||||
|
shares_outstanding_estimated=True,
|
||||||
|
latest_period_end=TODAY,
|
||||||
|
latest_filed_date=TODAY,
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(candidates.deriv, "derive", guarded)
|
||||||
|
candidate = (await candidates.build_candidates(session, today=TODAY))[0]
|
||||||
|
|
||||||
|
assert candidate.pe_ratio is None
|
||||||
|
assert candidate.market_cap == 50 * 123
|
||||||
|
assert candidate.revenue_growth == 7
|
||||||
|
assert candidate.unavailable_fields["pe_ratio"] == "split guard applied"
|
||||||
|
assert "weighted-average" in candidate.unavailable_fields["market_cap_estimated"]
|
||||||
@@ -12,6 +12,7 @@ from app.scheduler import (
|
|||||||
_last_successful,
|
_last_successful,
|
||||||
_run_shadow_import,
|
_run_shadow_import,
|
||||||
run_fundamentals_parity_report,
|
run_fundamentals_parity_report,
|
||||||
|
run_sec_fundamentals_import,
|
||||||
configure_scheduler,
|
configure_scheduler,
|
||||||
get_job_runtime_snapshot,
|
get_job_runtime_snapshot,
|
||||||
queue_backtest_options,
|
queue_backtest_options,
|
||||||
@@ -247,6 +248,95 @@ class TestShadowImportJobs:
|
|||||||
assert runtime["status"] == "skipped"
|
assert runtime["status"] == "skipped"
|
||||||
assert runtime["message"] == "Disabled"
|
assert runtime["message"] == "Disabled"
|
||||||
|
|
||||||
|
async def test_sec_failure_still_runs_activated_local_refresh(self, monkeypatch):
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
async def enabled(db, job_name):
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def unavailable(importer):
|
||||||
|
raise RuntimeError("SEC unavailable")
|
||||||
|
|
||||||
|
async def refreshed(db):
|
||||||
|
calls.append(db)
|
||||||
|
return {
|
||||||
|
"enabled": True,
|
||||||
|
"refreshed": 511,
|
||||||
|
"score_inputs_changed": 2,
|
||||||
|
"dimension_scores_staled": 2,
|
||||||
|
"composite_scores_staled": 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
||||||
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
|
monkeypatch.setattr("app.scheduler.run_import", unavailable)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
|
||||||
|
refreshed,
|
||||||
|
)
|
||||||
|
|
||||||
|
await run_sec_fundamentals_import()
|
||||||
|
|
||||||
|
assert len(calls) == 1
|
||||||
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
|
assert runtime["status"] == "error"
|
||||||
|
assert runtime["message"] == "SEC unavailable"
|
||||||
|
|
||||||
|
async def test_sec_success_surfaces_activated_refresh_summary(self, monkeypatch):
|
||||||
|
async def enabled(db, job_name):
|
||||||
|
return True
|
||||||
|
|
||||||
|
async def imported(importer):
|
||||||
|
return SimpleNamespace(
|
||||||
|
status="no_op", revision="abcdef1234567890", error_details=None
|
||||||
|
)
|
||||||
|
|
||||||
|
async def refreshed(db):
|
||||||
|
return {
|
||||||
|
"enabled": True,
|
||||||
|
"refreshed": 511,
|
||||||
|
"score_inputs_changed": 2,
|
||||||
|
"dimension_scores_staled": 2,
|
||||||
|
"composite_scores_staled": 2,
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
||||||
|
monkeypatch.setattr("app.scheduler._is_job_enabled", enabled)
|
||||||
|
monkeypatch.setattr("app.scheduler.run_import", imported)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
|
||||||
|
refreshed,
|
||||||
|
)
|
||||||
|
|
||||||
|
await run_sec_fundamentals_import()
|
||||||
|
|
||||||
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
|
assert runtime["status"] == "completed"
|
||||||
|
assert runtime["message"] == (
|
||||||
|
"no_op · abcdef123456 · cache 511 · 2 score inputs changed"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def test_disabled_sec_job_does_not_run_local_refresh(self, monkeypatch):
|
||||||
|
async def disabled(db, job_name):
|
||||||
|
return False
|
||||||
|
|
||||||
|
async def should_not_run(*args, **kwargs):
|
||||||
|
raise AssertionError("disabled SEC job ran work")
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.scheduler.async_session_factory", self._session_factory)
|
||||||
|
monkeypatch.setattr("app.scheduler._is_job_enabled", disabled)
|
||||||
|
monkeypatch.setattr("app.scheduler.run_import", should_not_run)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.scheduler.fundamental_data_refresh_service.refresh_if_enabled",
|
||||||
|
should_not_run,
|
||||||
|
)
|
||||||
|
|
||||||
|
await run_sec_fundamentals_import()
|
||||||
|
|
||||||
|
runtime = get_job_runtime_snapshot("sec_fundamentals_import")
|
||||||
|
assert runtime["status"] == "skipped"
|
||||||
|
assert runtime["message"] == "Disabled"
|
||||||
|
|
||||||
|
|
||||||
async def test_fundamentals_parity_job_surfaces_report_summary(monkeypatch):
|
async def test_fundamentals_parity_job_surfaces_report_summary(monkeypatch):
|
||||||
async def enabled(db, job_name):
|
async def enabled(db, job_name):
|
||||||
|
|||||||
Reference in New Issue
Block a user