feat: add fundamentals parity reporting
This commit is contained in:
@@ -61,6 +61,10 @@ class Settings(BaseSettings):
|
||||
sec_max_retries: int = 4
|
||||
sec_request_timeout_seconds: float = 30.0
|
||||
|
||||
# A5 read-only comparison artifacts. Production must keep this outside the
|
||||
# rsync deployment tree so the 5-7 day review window survives deploys.
|
||||
fundamentals_parity_report_dir: str = "reports/fundamentals-parity"
|
||||
|
||||
# Regime Monitor — FRED (VIX level + HY credit spreads). Optional: without it
|
||||
# the volatility (P5) and credit-spread (F2) signals are reported as n/a.
|
||||
fred_api_key: str = ""
|
||||
|
||||
@@ -453,6 +453,36 @@ async def toggle_job(
|
||||
)
|
||||
|
||||
|
||||
@router.get("/admin/fundamentals-parity", response_model=APIEnvelope)
|
||||
async def get_fundamentals_parity_report(
|
||||
_admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Latest read-only A5 source/score comparison, or null before first run."""
|
||||
return APIEnvelope(
|
||||
status="success", data=admin_service.get_fundamentals_parity_report()
|
||||
)
|
||||
|
||||
|
||||
@router.get("/admin/fundamentals-parity/csv", response_model=APIEnvelope)
|
||||
async def get_fundamentals_parity_csv(
|
||||
_admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Latest flattened A5 report for an authenticated browser download."""
|
||||
artifact = admin_service.get_fundamentals_parity_csv()
|
||||
data = None if artifact is None else {"filename": artifact[0], "content": artifact[1]}
|
||||
return APIEnvelope(status="success", data=data)
|
||||
|
||||
|
||||
@router.get("/admin/fundamentals-parity/json", response_model=APIEnvelope)
|
||||
async def get_fundamentals_parity_json(
|
||||
_admin: User = Depends(require_admin),
|
||||
):
|
||||
"""Canonical A5 JSON artifact for an authenticated browser download."""
|
||||
artifact = admin_service.get_fundamentals_parity_json()
|
||||
data = None if artifact is None else {"filename": artifact[0], "content": artifact[1]}
|
||||
return APIEnvelope(status="success", data=data)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System events (operational warnings / errors)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -40,6 +40,7 @@ from app.services import (
|
||||
sentiment_service,
|
||||
settings_store,
|
||||
shadow_book_service,
|
||||
fundamentals_parity_service,
|
||||
)
|
||||
from app.services.data_import import STATUS_FAILED, SourceImporter, run_import
|
||||
from app.services.dolt_earnings_importer import DoltEarningsImporter
|
||||
@@ -98,6 +99,7 @@ _JOB_NAMES = [
|
||||
"fundamental_collector",
|
||||
"dolt_earnings_import",
|
||||
"sec_fundamentals_import",
|
||||
"fundamentals_parity_report",
|
||||
"rr_scanner",
|
||||
"ticker_universe_sync",
|
||||
"alerts",
|
||||
@@ -981,6 +983,49 @@ async def run_sec_fundamentals_import() -> None:
|
||||
await _run_shadow_import("sec_fundamentals_import", SecFundamentalsImporter())
|
||||
|
||||
|
||||
async def run_fundamentals_parity_report() -> None:
|
||||
"""Generate the A5 comparison bundle without mutating live fundamentals/scores."""
|
||||
job_name = "fundamentals_parity_report"
|
||||
_log_event(logging.INFO, "job_start", job=job_name)
|
||||
_runtime_start(job_name, total=1)
|
||||
try:
|
||||
async with async_session_factory() as db:
|
||||
if not await _is_job_enabled(db, job_name):
|
||||
_runtime_finish(
|
||||
job_name, "skipped", processed=0, total=1, message="Disabled"
|
||||
)
|
||||
return
|
||||
report, artifacts = await fundamentals_parity_service.generate_and_store(
|
||||
db, settings.fundamentals_parity_report_dir
|
||||
)
|
||||
summary = report["summary"]
|
||||
message = (
|
||||
f"{summary['universe_count']} tickers · "
|
||||
f"{summary['fundamental_score_material_changes']} material score changes"
|
||||
)
|
||||
_runtime_finish(job_name, "completed", processed=1, total=1, message=message)
|
||||
_log_event(
|
||||
logging.INFO,
|
||||
"job_complete",
|
||||
job=job_name,
|
||||
generated_at=report["generated_at"],
|
||||
json_path=artifacts["json"],
|
||||
csv_path=artifacts["csv"],
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message="Cancelled")
|
||||
raise
|
||||
except Exception as exc:
|
||||
_runtime_finish(job_name, "error", processed=0, total=1, message=str(exc))
|
||||
_log_event(
|
||||
logging.ERROR,
|
||||
"job_error",
|
||||
job=job_name,
|
||||
error_type=type(exc).__name__,
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job: R:R Scanner
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -1524,6 +1569,7 @@ SCHEDULE_DEFAULTS: dict[str, str] = {
|
||||
# Shadow source imports. They never write legacy fundamental_data before A5.
|
||||
"schedule_dolt_earnings_cron": "30 2 * * *",
|
||||
"schedule_sec_fundamentals_cron": "0 4 * * *",
|
||||
"schedule_fundamentals_parity_cron": "30 5 * * *",
|
||||
# Fetch in-progress bars → scan → Telegram (manual MOC window).
|
||||
"schedule_near_close_pipeline_cron": "30 15 * * mon-fri",
|
||||
# Fetch final bars → outcome eval (must not run on the partial near-close bar).
|
||||
@@ -1539,6 +1585,7 @@ _CRON_JOBS: dict[str, str] = {
|
||||
"daily_pipeline": "schedule_daily_pipeline_cron",
|
||||
"dolt_earnings_import": "schedule_dolt_earnings_cron",
|
||||
"sec_fundamentals_import": "schedule_sec_fundamentals_cron",
|
||||
"fundamentals_parity_report": "schedule_fundamentals_parity_cron",
|
||||
"near_close_pipeline": "schedule_near_close_pipeline_cron",
|
||||
"after_close_pipeline": "schedule_after_close_pipeline_cron",
|
||||
"intraday_pipeline": "schedule_intraday_pipeline_cron",
|
||||
@@ -1645,6 +1692,17 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
name="SEC Fundamentals Import (shadow)",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
run_fundamentals_parity_report,
|
||||
_cron_trigger(
|
||||
cfg["schedule_fundamentals_parity_cron"],
|
||||
tz,
|
||||
"schedule_fundamentals_parity_cron",
|
||||
),
|
||||
id="fundamentals_parity_report",
|
||||
name="Fundamentals Parity Report (read-only)",
|
||||
replace_existing=True,
|
||||
)
|
||||
scheduler.add_job(
|
||||
run_near_close_pipeline,
|
||||
_cron_trigger(
|
||||
@@ -1720,6 +1778,9 @@ def configure_scheduler(schedule_config: dict[str, str] | None = None) -> None:
|
||||
},
|
||||
dolt_earnings_import={"cron": cfg["schedule_dolt_earnings_cron"]},
|
||||
sec_fundamentals_import={"cron": cfg["schedule_sec_fundamentals_cron"]},
|
||||
fundamentals_parity_report={
|
||||
"cron": cfg["schedule_fundamentals_parity_cron"]
|
||||
},
|
||||
near_close_pipeline={
|
||||
"cron": cfg["schedule_near_close_pipeline_cron"],
|
||||
"steps": [name for name, _ in _NEAR_CLOSE_PIPELINE_STEPS],
|
||||
|
||||
@@ -78,6 +78,9 @@ class ScheduleConfigUpdate(BaseModel):
|
||||
(min hour dom month dow); timezone is an IANA name (e.g. America/New_York)."""
|
||||
schedule_timezone: str | None = Field(default=None, max_length=64)
|
||||
schedule_daily_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_dolt_earnings_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_sec_fundamentals_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_fundamentals_parity_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_near_close_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_after_close_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||
schedule_intraday_pipeline_cron: str | None = Field(default=None, max_length=120)
|
||||
|
||||
@@ -26,6 +26,7 @@ class MetricItem(BaseModel):
|
||||
industry: MetricIndustry | None = None
|
||||
period_end: str | None = None
|
||||
filed_date: str | None = None
|
||||
caveat: str | None = None
|
||||
source: str = "sec"
|
||||
|
||||
|
||||
|
||||
@@ -614,6 +614,7 @@ VALID_JOB_NAMES = {
|
||||
"fundamental_collector",
|
||||
"dolt_earnings_import",
|
||||
"sec_fundamentals_import",
|
||||
"fundamentals_parity_report",
|
||||
"rr_scanner",
|
||||
"ticker_universe_sync",
|
||||
"outcome_evaluator",
|
||||
@@ -637,6 +638,7 @@ JOB_LABELS = {
|
||||
"fundamental_collector": "Fundamental Collector",
|
||||
"dolt_earnings_import": "Dolt Earnings Import (shadow)",
|
||||
"sec_fundamentals_import": "SEC Fundamentals Import (shadow)",
|
||||
"fundamentals_parity_report": "Fundamentals Parity Report (read-only)",
|
||||
"rr_scanner": "R:R Scanner",
|
||||
"ticker_universe_sync": "Ticker Universe Sync",
|
||||
"outcome_evaluator": "Outcome Evaluator",
|
||||
@@ -775,3 +777,30 @@ async def toggle_job(db: AsyncSession, job_name: str, enabled: bool) -> SystemSe
|
||||
|
||||
key = f"job_{job_name}_enabled"
|
||||
return await update_setting(db, key, str(enabled).lower())
|
||||
|
||||
|
||||
def get_fundamentals_parity_report() -> dict | None:
|
||||
"""Return the latest compact A5 summary, if the job has run."""
|
||||
from app.config import settings
|
||||
from app.services.fundamentals_parity_service import load_latest
|
||||
|
||||
report = load_latest(settings.fundamentals_parity_report_dir)
|
||||
if report is not None:
|
||||
report.pop("rows", None) # full per-ticker data is download-only
|
||||
return report
|
||||
|
||||
|
||||
def get_fundamentals_parity_csv() -> tuple[str, str] | None:
|
||||
"""Return the latest A5 CSV filename and content for authenticated download."""
|
||||
from app.config import settings
|
||||
from app.services.fundamentals_parity_service import load_latest_csv
|
||||
|
||||
return load_latest_csv(settings.fundamentals_parity_report_dir)
|
||||
|
||||
|
||||
def get_fundamentals_parity_json() -> tuple[str, str] | None:
|
||||
"""Return the canonical A5 JSON artifact for authenticated download."""
|
||||
from app.config import settings
|
||||
from app.services.fundamentals_parity_service import load_latest_json
|
||||
|
||||
return load_latest_json(settings.fundamentals_parity_report_dir)
|
||||
|
||||
@@ -121,6 +121,7 @@ def _build_metrics(derived, peer_derived, two: str | None) -> list[dict[str, Any
|
||||
"industry": industry,
|
||||
"period_end": _iso(series.period_end) if series else None,
|
||||
"filed_date": _iso(series.filed_date) if series else None,
|
||||
"caveat": series.caveat if series else None,
|
||||
"source": "sec",
|
||||
})
|
||||
return out
|
||||
@@ -303,7 +304,8 @@ async def _latest_close(db, ticker_id: int) -> tuple[float, date] | None:
|
||||
|
||||
def _empty_metrics() -> list[dict[str, Any]]:
|
||||
return [{"key": k, "value": None, "history": [], "industry": None,
|
||||
"period_end": None, "filed_date": None, "source": "sec"} for k in METRIC_KEYS]
|
||||
"period_end": None, "filed_date": None, "caveat": None,
|
||||
"source": "sec"} for k in METRIC_KEYS]
|
||||
|
||||
|
||||
def _empty_earnings() -> dict[str, Any]:
|
||||
|
||||
@@ -27,6 +27,11 @@ _FP_TO_Q = {"Q1": 1, "Q2": 2, "Q3": 3, "FY": 4}
|
||||
_Q_TO_FP = {1: "Q1", 2: "Q2", 3: "Q3", 4: "FY"}
|
||||
_PREV_FP = {"Q2": "Q1", "Q3": "Q2", "FY": "Q3"}
|
||||
TAPE_LEN = 4 # quarter-tape length
|
||||
SPLIT_SUSPECT_SHARE_CHANGE_PCT = 25.0
|
||||
SPLIT_SENSITIVE_CAVEAT = (
|
||||
"Not comparable: share count changed at least 25%; possible split or "
|
||||
"corporate action."
|
||||
)
|
||||
|
||||
# Duration (flow) fields differenced from YTD into discrete quarters + summed to TTM.
|
||||
_FLOW_FIELDS = (
|
||||
@@ -47,6 +52,7 @@ class MetricSeries:
|
||||
history: list[MetricPoint] = field(default_factory=list) # oldest -> newest, <= TAPE_LEN
|
||||
period_end: date | None = None
|
||||
filed_date: date | None = None
|
||||
caveat: str | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -96,6 +102,7 @@ def derive(snapshots: Iterable[Any]) -> DerivedFundamentals:
|
||||
"net_debt_to_ebitda": _leverage_series(selected, discrete, tape),
|
||||
"share_count_change_yoy": _share_change_series(selected, tape),
|
||||
}
|
||||
_guard_split_sensitive_metrics(result.metrics)
|
||||
for series in result.metrics.values():
|
||||
series.period_end = latest_row.period_end
|
||||
series.filed_date = latest_row.filed_date
|
||||
@@ -248,6 +255,40 @@ def _share_change_series(selected, tape) -> MetricSeries:
|
||||
return _series(pts)
|
||||
|
||||
|
||||
def _guard_split_sensitive_metrics(metrics: dict[str, MetricSeries]) -> None:
|
||||
"""Suppress historical comparisons likely distorted by a corporate action.
|
||||
|
||||
Company Facts has no point-in-time split factors. A large YoY share-count
|
||||
move can therefore make both the point-in-time share comparison and
|
||||
per-share EPS growth non-comparable. Keep the raw facts in snapshots, but
|
||||
expose nulls plus an explicit caveat in the user-facing derived series.
|
||||
"""
|
||||
shares = metrics.get("share_count_change_yoy")
|
||||
eps = metrics.get("eps_growth_yoy")
|
||||
if shares is None or eps is None:
|
||||
return
|
||||
|
||||
suspect_periods = {
|
||||
point.period_end
|
||||
for point in shares.history
|
||||
if point.value is not None
|
||||
and abs(point.value) >= SPLIT_SUSPECT_SHARE_CHANGE_PCT
|
||||
}
|
||||
if not suspect_periods:
|
||||
return
|
||||
|
||||
for series in (shares, eps):
|
||||
latest_guarded = bool(
|
||||
series.history and series.history[-1].period_end in suspect_periods
|
||||
)
|
||||
for point in series.history:
|
||||
if point.period_end in suspect_periods:
|
||||
point.value = None
|
||||
series.value = series.history[-1].value if series.history else None
|
||||
if latest_guarded:
|
||||
series.caveat = SPLIT_SENSITIVE_CAVEAT
|
||||
|
||||
|
||||
def _net_debt(row: Any) -> float | None:
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
@@ -0,0 +1,598 @@
|
||||
"""Read-only A5 comparison of legacy and SEC/Dolt fundamental inputs.
|
||||
|
||||
The report deliberately does not write ``fundamental_data`` or score tables.
|
||||
It reconstructs the current legacy and candidate fundamental scores, projects
|
||||
their composite-score/rank effect with the active weights, and archives a
|
||||
timestamped JSON + CSV bundle for explicit human approval.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import statistics
|
||||
from collections import defaultdict
|
||||
from datetime import date, datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.data_import_run import DataImportRun
|
||||
from app.models.earnings_event import EarningsEvent
|
||||
from app.models.fundamental import FundamentalData
|
||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.models.ohlcv import OHLCVRecord
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import fundamentals_derivation as deriv
|
||||
|
||||
REPORT_VERSION = 1
|
||||
APPROVAL_STATUS = "pending_explicit_approval"
|
||||
FIELD_KEYS = ("pe_ratio", "revenue_growth", "earnings_surprise")
|
||||
MIN_SCORE_METRICS = 2
|
||||
|
||||
# Materiality is a review aid, never an automatic cutover verdict. Definition
|
||||
# changes remain visible even when a delta falls inside these bands.
|
||||
FIELD_TOLERANCES = {
|
||||
"pe_ratio": {"absolute": 1.0, "relative_pct": 10.0},
|
||||
"revenue_growth": {"absolute": 2.0, "relative_pct": None},
|
||||
"earnings_surprise": {"absolute": 2.0, "relative_pct": None},
|
||||
}
|
||||
DEFINITION_NOTES = {
|
||||
"pe_ratio": (
|
||||
"Legacy provider P/E convention versus latest close divided by "
|
||||
"SEC-derived TTM diluted EPS."
|
||||
),
|
||||
"revenue_growth": (
|
||||
"Legacy provider growth convention versus SEC-derived TTM revenue YoY."
|
||||
),
|
||||
"earnings_surprise": (
|
||||
"Legacy provider latest surprise versus latest completed Dolt earnings "
|
||||
"event with actual and estimate."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def fundamental_score(
|
||||
pe_ratio: float | None,
|
||||
revenue_growth: float | None,
|
||||
earnings_surprise: float | None,
|
||||
) -> float | None:
|
||||
"""Match the production fundamental-dimension formula without persistence."""
|
||||
scores: list[float] = []
|
||||
if _finite(pe_ratio) and pe_ratio > 0:
|
||||
scores.append(max(0.0, min(100.0, 100.0 - (pe_ratio - 15.0) * (100.0 / 30.0))))
|
||||
if _finite(revenue_growth):
|
||||
scores.append(max(0.0, min(100.0, 50.0 + revenue_growth * 2.5)))
|
||||
if _finite(earnings_surprise):
|
||||
scores.append(max(0.0, min(100.0, 50.0 + earnings_surprise * 5.0)))
|
||||
return sum(scores) / len(scores) if len(scores) >= MIN_SCORE_METRICS else None
|
||||
|
||||
|
||||
async def build_report(
|
||||
db: AsyncSession,
|
||||
*,
|
||||
generated_at: datetime | None = None,
|
||||
today: date | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a point-in-time parity report from one database session."""
|
||||
generated_at = generated_at or datetime.now(timezone.utc)
|
||||
today = today or datetime.now(ZoneInfo("America/New_York")).date()
|
||||
|
||||
# A report must not mix rows from before and after a concurrent import
|
||||
# promotion. The scheduled job provides a fresh session, so establish the
|
||||
# production snapshot before its first query and have Postgres enforce the
|
||||
# no-write contract as well. SQLite tests retain their normal transaction.
|
||||
if db.get_bind().dialect.name == "postgresql":
|
||||
connection = await db.connection(
|
||||
execution_options={"isolation_level": "REPEATABLE READ"}
|
||||
)
|
||||
await connection.execute(text("SET TRANSACTION READ ONLY"))
|
||||
|
||||
tickers = list((await db.execute(select(Ticker).order_by(Ticker.symbol))).scalars())
|
||||
ticker_ids = [ticker.id for ticker in tickers]
|
||||
ciks = sorted({ticker.cik for ticker in tickers if ticker.cik})
|
||||
|
||||
legacy_by_ticker = await _legacy_values(db, ticker_ids)
|
||||
derived_by_cik = await _derived_by_cik(db, ciks)
|
||||
closes_by_ticker = await _latest_closes(db, ticker_ids)
|
||||
surprise_by_ticker = await _latest_surprises(db, ticker_ids, today)
|
||||
source_runs = await _source_runs(db)
|
||||
|
||||
rows: list[dict[str, Any]] = []
|
||||
for ticker in tickers:
|
||||
legacy = legacy_by_ticker.get(ticker.id)
|
||||
derived = derived_by_cik.get(ticker.cik) if ticker.cik else None
|
||||
close = closes_by_ticker.get(ticker.id)
|
||||
candidate_pe = (
|
||||
_pe(close[0], derived.ttm_diluted_eps)
|
||||
if close is not None and derived is not None
|
||||
else None
|
||||
)
|
||||
growth_series = (
|
||||
derived.metrics.get("revenue_growth_yoy") if derived is not None else None
|
||||
)
|
||||
candidate = {
|
||||
"pe_ratio": candidate_pe,
|
||||
"revenue_growth": growth_series.value if growth_series else None,
|
||||
"earnings_surprise": surprise_by_ticker.get(ticker.id),
|
||||
}
|
||||
legacy_values = {
|
||||
"pe_ratio": legacy.pe_ratio if legacy else None,
|
||||
"revenue_growth": legacy.revenue_growth if legacy else None,
|
||||
"earnings_surprise": legacy.earnings_surprise if legacy else None,
|
||||
}
|
||||
fields = {
|
||||
key: _field_comparison(key, legacy_values[key], candidate[key])
|
||||
for key in FIELD_KEYS
|
||||
}
|
||||
legacy_score = fundamental_score(**legacy_values)
|
||||
candidate_score = fundamental_score(**candidate)
|
||||
rows.append(
|
||||
{
|
||||
"symbol": ticker.symbol,
|
||||
"cik": ticker.cik,
|
||||
"legacy_fetched_at": _iso(legacy.fetched_at) if legacy else None,
|
||||
"price_date": _iso(close[1]) if close else None,
|
||||
"fields": fields,
|
||||
"scores": {
|
||||
"legacy_fundamental": _round(legacy_score),
|
||||
"candidate_fundamental": _round(candidate_score),
|
||||
"fundamental_delta": _delta(legacy_score, candidate_score),
|
||||
"legacy_fundamental_rank": None,
|
||||
"candidate_fundamental_rank": None,
|
||||
"fundamental_rank_change": None,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
_attach_ranks(rows, "legacy_fundamental", "legacy_fundamental_rank")
|
||||
_attach_ranks(rows, "candidate_fundamental", "candidate_fundamental_rank")
|
||||
for row in rows:
|
||||
scores = row["scores"]
|
||||
scores["fundamental_rank_change"] = _rank_change(
|
||||
scores["legacy_fundamental_rank"], scores["candidate_fundamental_rank"]
|
||||
)
|
||||
|
||||
return {
|
||||
"report_version": REPORT_VERSION,
|
||||
"generated_at": generated_at.isoformat(),
|
||||
"as_of_date": today.isoformat(),
|
||||
"approval_status": APPROVAL_STATUS,
|
||||
"read_only": True,
|
||||
"fundamental_score_formula": (
|
||||
"Equal-weighted mean of 2+ available sub-scores: P/E = "
|
||||
"clamp(100-(pe-15)*(100/30)); revenue growth = "
|
||||
"clamp(50+growth*2.5); earnings surprise = "
|
||||
"clamp(50+surprise*5)."
|
||||
),
|
||||
"source_runs": source_runs,
|
||||
"definition_notes": DEFINITION_NOTES,
|
||||
"materiality_notes": {
|
||||
"fields": FIELD_TOLERANCES,
|
||||
"fundamental_score_absolute": 5.0,
|
||||
"automatic_cutover": False,
|
||||
},
|
||||
"summary": _summary(rows),
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
|
||||
def store_report(report: dict[str, Any], report_dir: str | Path) -> dict[str, str]:
|
||||
"""Atomically archive JSON/CSV artifacts and update the latest manifest."""
|
||||
directory = Path(report_dir).expanduser().resolve()
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
stamp = _artifact_stamp(report["generated_at"])
|
||||
json_name = f"fundamentals-parity-{stamp}.json"
|
||||
csv_name = f"fundamentals-parity-{stamp}.csv"
|
||||
json_path = directory / json_name
|
||||
csv_path = directory / csv_name
|
||||
|
||||
_atomic_write(json_path, json.dumps(report, indent=2, sort_keys=True) + "\n")
|
||||
_atomic_write(csv_path, report_csv(report))
|
||||
manifest = {
|
||||
"generated_at": report["generated_at"],
|
||||
"json_file": json_name,
|
||||
"csv_file": csv_name,
|
||||
}
|
||||
_atomic_write(
|
||||
directory / "latest.json",
|
||||
json.dumps(manifest, indent=2, sort_keys=True) + "\n",
|
||||
)
|
||||
return {
|
||||
"json": str(json_path),
|
||||
"csv": str(csv_path),
|
||||
"manifest": str(directory / "latest.json"),
|
||||
}
|
||||
|
||||
|
||||
async def generate_and_store(
|
||||
db: AsyncSession,
|
||||
report_dir: str | Path,
|
||||
*,
|
||||
generated_at: datetime | None = None,
|
||||
today: date | None = None,
|
||||
) -> tuple[dict[str, Any], dict[str, str]]:
|
||||
report = await build_report(db, generated_at=generated_at, today=today)
|
||||
return report, store_report(report, report_dir)
|
||||
|
||||
|
||||
def load_latest(report_dir: str | Path) -> dict[str, Any] | None:
|
||||
manifest = _load_manifest(report_dir)
|
||||
if manifest is None:
|
||||
return None
|
||||
try:
|
||||
path = _manifest_artifact(report_dir, manifest, "json_file")
|
||||
loaded = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
return loaded if isinstance(loaded, dict) else None
|
||||
|
||||
|
||||
def load_latest_csv(report_dir: str | Path) -> tuple[str, str] | None:
|
||||
return _load_latest_text_artifact(report_dir, "csv_file")
|
||||
|
||||
|
||||
def load_latest_json(report_dir: str | Path) -> tuple[str, str] | None:
|
||||
return _load_latest_text_artifact(report_dir, "json_file")
|
||||
|
||||
|
||||
def _load_latest_text_artifact(
|
||||
report_dir: str | Path, manifest_key: str
|
||||
) -> tuple[str, str] | None:
|
||||
manifest = _load_manifest(report_dir)
|
||||
if manifest is None:
|
||||
return None
|
||||
try:
|
||||
path = _manifest_artifact(report_dir, manifest, manifest_key)
|
||||
return path.name, path.read_text(encoding="utf-8")
|
||||
except (OSError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def report_csv(report: dict[str, Any]) -> str:
|
||||
output = io.StringIO(newline="")
|
||||
columns = [
|
||||
"symbol",
|
||||
"cik",
|
||||
"legacy_fetched_at",
|
||||
"price_date",
|
||||
*(
|
||||
f"{field}_{suffix}"
|
||||
for field in FIELD_KEYS
|
||||
for suffix in ("legacy", "candidate", "absolute_delta", "relative_delta_pct", "material")
|
||||
),
|
||||
"legacy_fundamental",
|
||||
"candidate_fundamental",
|
||||
"fundamental_delta",
|
||||
"legacy_fundamental_rank",
|
||||
"candidate_fundamental_rank",
|
||||
"fundamental_rank_change",
|
||||
]
|
||||
writer = csv.DictWriter(output, fieldnames=columns)
|
||||
writer.writeheader()
|
||||
for row in report.get("rows", []):
|
||||
flat = {
|
||||
"symbol": row["symbol"],
|
||||
"cik": row.get("cik"),
|
||||
"legacy_fetched_at": row.get("legacy_fetched_at"),
|
||||
"price_date": row.get("price_date"),
|
||||
**row["scores"],
|
||||
}
|
||||
for field in FIELD_KEYS:
|
||||
comparison = row["fields"][field]
|
||||
for suffix in (
|
||||
"legacy",
|
||||
"candidate",
|
||||
"absolute_delta",
|
||||
"relative_delta_pct",
|
||||
"material",
|
||||
):
|
||||
flat[f"{field}_{suffix}"] = comparison.get(suffix)
|
||||
writer.writerow(flat)
|
||||
return output.getvalue()
|
||||
|
||||
|
||||
async def _legacy_values(
|
||||
db: AsyncSession, ticker_ids: list[int]
|
||||
) -> dict[int, FundamentalData]:
|
||||
if not ticker_ids:
|
||||
return {}
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(FundamentalData).where(FundamentalData.ticker_id.in_(ticker_ids))
|
||||
)
|
||||
).scalars()
|
||||
return {row.ticker_id: row for row in rows}
|
||||
|
||||
|
||||
async def _derived_by_cik(
|
||||
db: AsyncSession, ciks: list[str]
|
||||
) -> dict[str, deriv.DerivedFundamentals]:
|
||||
if not ciks:
|
||||
return {}
|
||||
grouped: dict[str, list[FundamentalSnapshot]] = defaultdict(list)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(FundamentalSnapshot).where(FundamentalSnapshot.cik.in_(ciks))
|
||||
)
|
||||
).scalars()
|
||||
for row in rows:
|
||||
grouped[row.cik].append(row)
|
||||
return {cik: deriv.derive(grouped.get(cik, [])) for cik in ciks}
|
||||
|
||||
|
||||
async def _latest_closes(
|
||||
db: AsyncSession, ticker_ids: list[int]
|
||||
) -> dict[int, tuple[float, date]]:
|
||||
if not ticker_ids:
|
||||
return {}
|
||||
latest = (
|
||||
select(OHLCVRecord.ticker_id, func.max(OHLCVRecord.date).label("max_date"))
|
||||
.where(OHLCVRecord.ticker_id.in_(ticker_ids))
|
||||
.group_by(OHLCVRecord.ticker_id)
|
||||
.subquery()
|
||||
)
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(OHLCVRecord.ticker_id, OHLCVRecord.close, OHLCVRecord.date).join(
|
||||
latest,
|
||||
(OHLCVRecord.ticker_id == latest.c.ticker_id)
|
||||
& (OHLCVRecord.date == latest.c.max_date),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return {
|
||||
ticker_id: (float(close), close_date)
|
||||
for ticker_id, close, close_date in rows
|
||||
if _finite(close)
|
||||
}
|
||||
|
||||
|
||||
async def _latest_surprises(
|
||||
db: AsyncSession, ticker_ids: list[int], today: date
|
||||
) -> dict[int, float]:
|
||||
if not ticker_ids:
|
||||
return {}
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(EarningsEvent)
|
||||
.where(
|
||||
EarningsEvent.ticker_id.in_(ticker_ids),
|
||||
EarningsEvent.announce_date < today,
|
||||
)
|
||||
.order_by(EarningsEvent.ticker_id, EarningsEvent.announce_date.desc())
|
||||
)
|
||||
).scalars()
|
||||
out: dict[int, float] = {}
|
||||
for row in rows:
|
||||
if row.ticker_id in out:
|
||||
continue
|
||||
surprise = _surprise(row.eps_estimate, row.eps_actual)
|
||||
if surprise is not None:
|
||||
out[row.ticker_id] = surprise
|
||||
return out
|
||||
|
||||
|
||||
async def _source_runs(db: AsyncSession) -> dict[str, dict[str, Any] | None]:
|
||||
sources = ("sec_facts", "dolt_earnings")
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(DataImportRun)
|
||||
.where(
|
||||
DataImportRun.source.in_(sources),
|
||||
DataImportRun.status.in_(("promoted", "no_op")),
|
||||
)
|
||||
.order_by(DataImportRun.id.desc())
|
||||
)
|
||||
).scalars()
|
||||
latest: dict[str, dict[str, Any] | None] = {source: None for source in sources}
|
||||
for row in rows:
|
||||
if latest[row.source] is None:
|
||||
latest[row.source] = {
|
||||
"run_id": row.id,
|
||||
"status": row.status,
|
||||
"revision": row.revision,
|
||||
"source_max_date": _iso(row.source_max_date),
|
||||
"completed_at": _iso(row.completed_at),
|
||||
}
|
||||
return latest
|
||||
|
||||
|
||||
def _field_comparison(
|
||||
key: str, legacy: float | None, candidate: float | None
|
||||
) -> dict[str, Any]:
|
||||
legacy = float(legacy) if _finite(legacy) else None
|
||||
candidate = float(candidate) if _finite(candidate) else None
|
||||
absolute = _delta(legacy, candidate)
|
||||
relative = (
|
||||
None
|
||||
if absolute is None or legacy in (None, 0)
|
||||
else round(absolute / abs(legacy) * 100.0, 4)
|
||||
)
|
||||
tolerance = FIELD_TOLERANCES[key]
|
||||
material = False
|
||||
if absolute is not None:
|
||||
material = abs(absolute) > tolerance["absolute"]
|
||||
relative_limit = tolerance["relative_pct"]
|
||||
if relative_limit is not None:
|
||||
material = material and relative is not None and abs(relative) > relative_limit
|
||||
return {
|
||||
"legacy": _round(legacy),
|
||||
"candidate": _round(candidate),
|
||||
"absolute_delta": absolute,
|
||||
"relative_delta_pct": relative,
|
||||
"material": material,
|
||||
"definition_changed": True,
|
||||
}
|
||||
|
||||
|
||||
def _attach_ranks(rows: list[dict[str, Any]], value_key: str, rank_key: str) -> None:
|
||||
values = [
|
||||
row["scores"][value_key]
|
||||
for row in rows
|
||||
if _finite(row["scores"][value_key])
|
||||
]
|
||||
for row in rows:
|
||||
value = row["scores"][value_key]
|
||||
row["scores"][rank_key] = (
|
||||
1 + sum(other > value for other in values) if _finite(value) else None
|
||||
)
|
||||
|
||||
|
||||
def _summary(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
field_stats = {}
|
||||
for key in FIELD_KEYS:
|
||||
comparisons = [row["fields"][key] for row in rows]
|
||||
deltas = [
|
||||
abs(item["absolute_delta"])
|
||||
for item in comparisons
|
||||
if item["absolute_delta"] is not None
|
||||
]
|
||||
field_stats[key] = {
|
||||
"legacy_available": sum(item["legacy"] is not None for item in comparisons),
|
||||
"candidate_available": sum(
|
||||
item["candidate"] is not None for item in comparisons
|
||||
),
|
||||
"both_available": len(deltas),
|
||||
"material_differences": sum(item["material"] for item in comparisons),
|
||||
"median_absolute_delta": _round(statistics.median(deltas) if deltas else None),
|
||||
"p95_absolute_delta": _round(_percentile(deltas, 0.95)),
|
||||
"max_absolute_delta": _round(max(deltas) if deltas else None),
|
||||
}
|
||||
|
||||
fundamental_deltas = _score_deltas(rows, "fundamental_delta")
|
||||
changed_rows = sorted(
|
||||
(
|
||||
{
|
||||
"symbol": row["symbol"],
|
||||
"fundamental_delta": row["scores"]["fundamental_delta"],
|
||||
"fundamental_rank_change": row["scores"]["fundamental_rank_change"],
|
||||
}
|
||||
for row in rows
|
||||
if row["scores"]["fundamental_delta"] is not None
|
||||
),
|
||||
key=lambda item: (
|
||||
abs(item["fundamental_delta"] or 0),
|
||||
),
|
||||
reverse=True,
|
||||
)[:20]
|
||||
return {
|
||||
"universe_count": len(rows),
|
||||
"legacy_fundamental_score_available": _count_score(
|
||||
rows, "legacy_fundamental"
|
||||
),
|
||||
"candidate_fundamental_score_available": _count_score(
|
||||
rows, "candidate_fundamental"
|
||||
),
|
||||
"fundamental_scores_compared": len(fundamental_deltas),
|
||||
"fundamental_score_material_changes": sum(
|
||||
abs(delta) > 5.0 for delta in fundamental_deltas
|
||||
),
|
||||
"fundamental_rank_changes": _rank_change_count(
|
||||
rows, "fundamental_rank_change"
|
||||
),
|
||||
"field_stats": field_stats,
|
||||
"largest_changes": changed_rows,
|
||||
}
|
||||
|
||||
|
||||
def _score_deltas(rows: Iterable[dict[str, Any]], key: str) -> list[float]:
|
||||
return [
|
||||
row["scores"][key]
|
||||
for row in rows
|
||||
if row["scores"][key] is not None
|
||||
]
|
||||
|
||||
|
||||
def _count_score(rows: Iterable[dict[str, Any]], key: str) -> int:
|
||||
return sum(row["scores"][key] is not None for row in rows)
|
||||
|
||||
|
||||
def _rank_change_count(rows: Iterable[dict[str, Any]], key: str) -> int:
|
||||
return sum(
|
||||
row["scores"][key] not in (None, 0)
|
||||
for row in rows
|
||||
)
|
||||
|
||||
|
||||
def _rank_change(legacy: int | None, candidate: int | None) -> int | None:
|
||||
# Positive means the candidate improved its rank.
|
||||
return legacy - candidate if legacy is not None and candidate is not None else None
|
||||
|
||||
|
||||
def _surprise(estimate: float | None, actual: float | None) -> float | None:
|
||||
if not _finite(estimate) or not _finite(actual) or estimate == 0:
|
||||
return None
|
||||
return (actual - estimate) / abs(estimate) * 100.0
|
||||
|
||||
|
||||
def _pe(price: float | None, ttm_eps: float | None) -> float | None:
|
||||
if not _finite(price) or price <= 0 or not _finite(ttm_eps) or ttm_eps <= 0:
|
||||
return None
|
||||
return price / ttm_eps
|
||||
|
||||
|
||||
def _delta(legacy: float | None, candidate: float | None) -> float | None:
|
||||
if not _finite(legacy) or not _finite(candidate):
|
||||
return None
|
||||
return round(candidate - legacy, 4)
|
||||
|
||||
|
||||
def _round(value: float | None, digits: int = 4) -> float | None:
|
||||
return round(float(value), digits) if _finite(value) else None
|
||||
|
||||
|
||||
def _percentile(values: list[float], quantile: float) -> float | None:
|
||||
if not values:
|
||||
return None
|
||||
ordered = sorted(values)
|
||||
index = max(0, math.ceil(quantile * len(ordered)) - 1)
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def _finite(value: Any) -> bool:
|
||||
return (
|
||||
isinstance(value, (int, float))
|
||||
and not isinstance(value, bool)
|
||||
and math.isfinite(value)
|
||||
)
|
||||
|
||||
|
||||
def _iso(value: Any) -> str | None:
|
||||
return value.isoformat() if value is not None else None
|
||||
|
||||
|
||||
def _artifact_stamp(raw: str) -> str:
|
||||
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
return parsed.astimezone(timezone.utc).strftime("%Y%m%dT%H%M%S%fZ")
|
||||
|
||||
|
||||
def _atomic_write(path: Path, content: str) -> None:
|
||||
temp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
||||
temp.write_text(content, encoding="utf-8", newline="")
|
||||
os.replace(temp, path)
|
||||
|
||||
|
||||
def _load_manifest(report_dir: str | Path) -> dict[str, Any] | None:
|
||||
path = Path(report_dir).expanduser().resolve() / "latest.json"
|
||||
try:
|
||||
loaded = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError, TypeError, ValueError):
|
||||
return None
|
||||
return loaded if isinstance(loaded, dict) else None
|
||||
|
||||
|
||||
def _manifest_artifact(
|
||||
report_dir: str | Path, manifest: dict[str, Any], key: str
|
||||
) -> Path:
|
||||
directory = Path(report_dir).expanduser().resolve()
|
||||
name = Path(str(manifest.get(key, ""))).name
|
||||
if not name:
|
||||
raise ValueError(f"Latest parity manifest has no {key}")
|
||||
return directory / name
|
||||
Reference in New Issue
Block a user