feat(sec): A3 slice 2b — SEC fundamentals importer (shadow ingestion)
SecFundamentalsImporter (SourceImporter, source=sec_facts): populates immutable fundamental_snapshots from Company Facts and back-fills tickers.cik/sic, driven by the EDGAR daily index. Shadow only. Guardrails per review: - detect_revision caches the resolved universe + exact tracked index rows and composes the revision from them; stage consumes those same cached inputs (no index/universe refetch) so promoted data matches the computed revision. - Resolution is read-only in stage (proposals only); ticker writes happen in promote via apply_ticker_updates. - validate runs the index<->Company-Facts consistency gate before any write: a tracked XBRL index accession missing from Company Facts fails the run (they lag independently) so we retry, not record null. Non-XBRL amendments are skipped with a recorded reason. Backfill has a coverage floor. - promote inserts ON CONFLICT (accession) DO NOTHING (immutable), reports differing existing accessions without mutating, and applies ticker updates in the same transaction. - Full-history backfill on first run / for newly-added issuers (include_history); incremental fetch only for issuers that filed. Parser: split parse result into skipped_filings vs field_issues (coverage must not count field warnings); header notes the us-gaap shares fallback; added companyfacts_accessions() for the gate. Verified live end-to-end (AAPL + GOOGL backfill): 112 snapshots, cik/sic set, GOOGL shares via us-gaap fallback, AAPL via dei. Tests: 6 importer (backfill, incremental, consistency-gate fail, non-XBRL skip, read-only-on-failure, conflict-discrepancy) + parser ParseResult updates. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -11,17 +11,24 @@ The load-bearing rules (design Decision 2 + review):
|
||||
- Duration facts are stored as **cumulative YTD**: pick the fact whose span
|
||||
matches the fiscal-period-to-date length (Q1≈3mo … FY≈12mo) within tolerance.
|
||||
If no YTD-length fact exists, store null — never a discrete masquerading as YTD.
|
||||
- Balance-sheet instants are taken at `end == reportDate`; `shares_outstanding`
|
||||
is the cover-page `dei` fact whose own `end` (cover date) is stored separately.
|
||||
- Balance-sheet instants are taken at `end == reportDate`. `shares_outstanding`
|
||||
is a single consolidated value: the cover-page `dei` fact (its own cover-date
|
||||
`end` stored separately) if present, else `us-gaap:CommonStockSharesOutstanding`
|
||||
at period end (e.g. Alphabet has no `dei` fact) — never a class sum or the
|
||||
weighted-average/diluted count.
|
||||
- Cash and debt composites are aggregate-first and mutually exclusive (each
|
||||
source tag counted at most once).
|
||||
|
||||
`parse_snapshots` separates `skipped_filings` (no usable row produced) from
|
||||
`field_issues` (a row was produced but a field is null/ambiguous) — callers must
|
||||
not treat field issues as missing coverage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
@@ -107,34 +114,49 @@ class FilingMeta:
|
||||
form: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParseResult:
|
||||
rows: list[SnapshotRow] = field(default_factory=list)
|
||||
# accessions for which NO row was produced (no facts / no usable period).
|
||||
skipped_filings: list[dict[str, str]] = field(default_factory=list)
|
||||
# accessions with a row but a field-level warning (e.g. ambiguous shares).
|
||||
field_issues: list[dict[str, str]] = field(default_factory=list)
|
||||
|
||||
|
||||
def parse_snapshots(
|
||||
companyfacts: dict[str, Any],
|
||||
filings: dict[str, FilingMeta],
|
||||
accessions: set[str],
|
||||
) -> tuple[list[SnapshotRow], list[dict[str, str]]]:
|
||||
) -> ParseResult:
|
||||
"""Build snapshot rows for ``accessions`` (those with facts + filing meta).
|
||||
|
||||
Returns (rows, skips) where each skip is {accession, reason} for filings
|
||||
with no usable period identity — the caller counts these in validation_json.
|
||||
``skipped_filings`` = no row produced (missing facts/meta or no usable period
|
||||
identity); ``field_issues`` = a row was produced but a field is null/ambiguous.
|
||||
Callers must not use field issues as failed-row coverage.
|
||||
"""
|
||||
cik = f"{int(companyfacts['cik']):010d}"
|
||||
by_accn = _index_by_accession(companyfacts)
|
||||
rows: list[SnapshotRow] = []
|
||||
skips: list[dict[str, str]] = []
|
||||
result = ParseResult()
|
||||
for accn in accessions:
|
||||
meta = filings.get(accn)
|
||||
facts = by_accn.get(accn)
|
||||
if meta is None or not facts:
|
||||
skips.append({"accession": accn, "reason": "no facts or filing metadata"})
|
||||
result.skipped_filings.append({"accession": accn, "reason": "no facts or filing metadata"})
|
||||
continue
|
||||
row, note = _parse_one(cik, accn, facts, meta)
|
||||
if row is None:
|
||||
skips.append({"accession": accn, "reason": note or "unparseable"})
|
||||
result.skipped_filings.append({"accession": accn, "reason": note or "unparseable"})
|
||||
continue
|
||||
rows.append(row)
|
||||
if note: # row produced, but a field-level issue to count in validation
|
||||
skips.append({"accession": accn, "reason": note})
|
||||
return rows, skips
|
||||
result.rows.append(row)
|
||||
if note:
|
||||
result.field_issues.append({"accession": accn, "reason": note})
|
||||
return result
|
||||
|
||||
|
||||
def companyfacts_accessions(companyfacts: dict[str, Any]) -> set[str]:
|
||||
"""Every accession that appears anywhere in a companyfacts payload — used by
|
||||
the importer's index↔Company-Facts consistency gate."""
|
||||
return set(_index_by_accession(companyfacts).keys())
|
||||
|
||||
|
||||
def _index_by_accession(companyfacts: dict[str, Any]) -> dict[str, list[Fact]]:
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
"""SEC fundamentals importer (workstream A, phase A3).
|
||||
|
||||
A ``SourceImporter`` (see ``app/services/data_import.py``) that populates the
|
||||
immutable ``fundamental_snapshots`` from SEC Company Facts and back-fills
|
||||
``tickers.cik/sic/sic_description``. EDGAR-daily-index driven: it fetches
|
||||
companyfacts only for tracked issuers that filed since the last run (full-history
|
||||
backfill on the first run / for newly-added issuers). Shadow only — nothing reads
|
||||
snapshots until A4.
|
||||
|
||||
Guardrails (design + reviews):
|
||||
- ``detect_revision`` caches the resolved universe + the exact tracked index rows
|
||||
and composes the revision from them; ``stage`` consumes those same cached inputs
|
||||
(it does not refetch the index/universe) so promoted data always matches the
|
||||
computed revision.
|
||||
- Resolution is read-only in ``stage`` (proposals only); ticker writes happen in
|
||||
``promote`` via ``apply_ticker_updates``.
|
||||
- ``validate`` runs the **index↔Company-Facts consistency gate** before any write:
|
||||
a tracked XBRL index accession missing from Company Facts fails the run (the two
|
||||
are separate SEC products that can lag) so we retry rather than record a
|
||||
null/partial snapshot. Non-XBRL amendments are skipped with a recorded reason.
|
||||
- ``promote`` inserts snapshots ``ON CONFLICT (accession) DO NOTHING`` (immutable),
|
||||
reports differing existing accessions, and applies ticker updates in the same
|
||||
transaction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from typing import Any, Callable
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from app.database import insert_for_session
|
||||
from app.models.data_import_run import DataImportRun
|
||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.services import sec_facts_parser as parser
|
||||
from app.services import sec_universe
|
||||
from app.services.data_import import STATUS_PROMOTED, ValidationResult
|
||||
from app.services.sec_client import SecClient, SecError, cik10
|
||||
from app.services.sec_facts_parser import FilingMeta, SnapshotRow
|
||||
from app.services.sec_universe import ResolvedUniverse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = "sec_facts"
|
||||
_XBRL_FORMS = {"10-K", "10-Q", "10-K/A", "10-Q/A"}
|
||||
# On the one-time backfill, require this fraction of tracked issuers to yield at
|
||||
# least one snapshot (guards a broken fetch/parse from promoting a hollow table).
|
||||
MIN_BACKFILL_COVERAGE = 0.5
|
||||
# Bound how far back the incremental index walk goes if the job hasn't run in a
|
||||
# while (each day = one small request); older gaps are logged, not silently lost.
|
||||
_MAX_INDEX_WALK_DAYS = 45
|
||||
|
||||
_SNAPSHOT_COLS = (
|
||||
"cik", "accession", "form", "filed_date", "accepted_at", "period_start",
|
||||
"period_end", "fiscal_year", "fiscal_period", "revenue", "net_income",
|
||||
"operating_income", "diluted_eps", "cfo", "capex", "depreciation_amortization",
|
||||
"cash_and_st_investments", "total_debt", "shares_outstanding",
|
||||
"shares_outstanding_date",
|
||||
)
|
||||
# Fields compared to flag a differing existing accession (immutable → report, not mutate).
|
||||
_DISCREPANCY_COLS = ("period_end", "fiscal_year", "fiscal_period", "revenue", "net_income")
|
||||
|
||||
|
||||
@dataclass
|
||||
class StagedFundamentals:
|
||||
resolved: ResolvedUniverse
|
||||
sic_updates: list[tuple[int, str | None, str | None]] = field(default_factory=list)
|
||||
rows: list[SnapshotRow] = field(default_factory=list)
|
||||
skipped_filings: list[dict[str, str]] = field(default_factory=list)
|
||||
field_issues: list[dict[str, str]] = field(default_factory=list)
|
||||
skipped_non_xbrl: list[dict[str, str]] = field(default_factory=list)
|
||||
missing_xbrl: list[dict[str, str]] = field(default_factory=list)
|
||||
backfill: bool = False
|
||||
issuers_fetched: int = 0
|
||||
issuers_with_rows: int = 0
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class SecFundamentalsImporter:
|
||||
source = SOURCE
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client_factory: Callable[[], SecClient] | None = None,
|
||||
today: date | None = None,
|
||||
) -> None:
|
||||
self._client_factory = client_factory or (lambda: SecClient())
|
||||
self.today = today or _now().date()
|
||||
# cached by detect_revision, consumed by stage:
|
||||
self._resolved: ResolvedUniverse | None = None
|
||||
self._index_rows: list[dict[str, Any]] = []
|
||||
self._latest_index_date: date | None = None
|
||||
self._backfill = False
|
||||
|
||||
# -- SourceImporter protocol -------------------------------------------
|
||||
|
||||
async def detect_revision(self, db) -> str | None:
|
||||
async with self._client_factory() as client:
|
||||
self._resolved = await sec_universe.resolve_ciks(db, client)
|
||||
last_processed = await self._last_processed_index_date(db)
|
||||
self._latest_index_date = await client.latest_index_date(self.today)
|
||||
if self._latest_index_date is None:
|
||||
raise SecError("no EDGAR daily index available")
|
||||
if last_processed is None:
|
||||
self._backfill = True
|
||||
self._index_rows = []
|
||||
else:
|
||||
self._backfill = False
|
||||
self._index_rows = await self._collect_index_rows(
|
||||
client, last_processed, self._latest_index_date
|
||||
)
|
||||
content = sec_universe.index_content_hash(self._index_rows)
|
||||
return sec_universe.compose_revision(
|
||||
self._latest_index_date, content, self._resolved.symbol_to_cik
|
||||
)
|
||||
|
||||
async def stage(self, db) -> StagedFundamentals:
|
||||
assert self._resolved is not None, "detect_revision must run first"
|
||||
resolved = self._resolved
|
||||
staged = StagedFundamentals(resolved=resolved, backfill=self._backfill)
|
||||
|
||||
cik_to_tids = resolved.cik_to_ticker_ids
|
||||
filed_by_cik: dict[int, list[str]] = defaultdict(list)
|
||||
for r in self._index_rows:
|
||||
if r["cik"] in cik_to_tids:
|
||||
filed_by_cik[r["cik"]].append(r["accession"])
|
||||
|
||||
existing = await self._ciks_with_snapshots(db, set(cik_to_tids))
|
||||
if self._backfill:
|
||||
backfill_ciks = set(cik_to_tids)
|
||||
else:
|
||||
# Newly added issuers (resolved but no snapshots yet) get a full-history
|
||||
# backfill; issuers that already have history are handled incrementally.
|
||||
backfill_ciks = {c for c in cik_to_tids if c not in existing}
|
||||
incremental_ciks = set(filed_by_cik) - backfill_ciks
|
||||
|
||||
async with self._client_factory() as client:
|
||||
for cik in sorted(backfill_ciks | incremental_ciks):
|
||||
is_backfill = cik in backfill_ciks
|
||||
await self._stage_issuer(client, cik, is_backfill, filed_by_cik, staged)
|
||||
return staged
|
||||
|
||||
async def _stage_issuer(self, client, cik, is_backfill, filed_by_cik, staged) -> None:
|
||||
cf = await client.companyfacts(cik)
|
||||
sub = await client.submissions(cik, include_history=is_backfill)
|
||||
xbrl_meta, nonxbrl = _filing_meta(sub)
|
||||
|
||||
if is_backfill:
|
||||
accns = set(xbrl_meta)
|
||||
else:
|
||||
present = parser.companyfacts_accessions(cf)
|
||||
accns = set()
|
||||
for accn in filed_by_cik.get(cik, []):
|
||||
if accn in nonxbrl:
|
||||
staged.skipped_non_xbrl.append({"cik": cik10(cik), "accession": accn})
|
||||
elif accn in xbrl_meta and accn in present:
|
||||
accns.add(accn)
|
||||
else:
|
||||
# XBRL (or unknown) filing not yet in Company Facts → the products
|
||||
# have lagged; fail+retry rather than record nothing for it.
|
||||
staged.missing_xbrl.append({"cik": cik10(cik), "accession": accn})
|
||||
|
||||
result = parser.parse_snapshots(cf, xbrl_meta, accns)
|
||||
staged.rows.extend(result.rows)
|
||||
staged.skipped_filings.extend(result.skipped_filings)
|
||||
staged.field_issues.extend(result.field_issues)
|
||||
staged.issuers_fetched += 1
|
||||
if result.rows:
|
||||
staged.issuers_with_rows += 1
|
||||
|
||||
# SIC proposal for this issuer's tickers (read-only; applied in promote).
|
||||
sic = str(sub["sic"]) if sub.get("sic") else None
|
||||
desc = sub.get("sic_description")
|
||||
for tid in staged.resolved.cik_to_ticker_ids.get(cik, []):
|
||||
staged.sic_updates.append((tid, sic, desc))
|
||||
|
||||
async def validate(self, db, staged: StagedFundamentals) -> ValidationResult:
|
||||
messages: list[str] = []
|
||||
|
||||
# Consistency gate — before any write.
|
||||
if staged.missing_xbrl:
|
||||
messages.append(
|
||||
f"{len(staged.missing_xbrl)} tracked XBRL filing(s) not yet in "
|
||||
"Company Facts (index/facts lag) — retry"
|
||||
)
|
||||
|
||||
accns = [r.accession for r in staged.rows]
|
||||
if len(accns) != len(set(accns)):
|
||||
messages.append("duplicate accession in staged snapshots")
|
||||
|
||||
if staged.backfill:
|
||||
n_issuers = len(staged.resolved.cik_to_ticker_ids)
|
||||
coverage = staged.issuers_with_rows / n_issuers if n_issuers else 0.0
|
||||
if coverage < MIN_BACKFILL_COVERAGE:
|
||||
messages.append(
|
||||
f"backfill coverage {coverage:.0%} < {MIN_BACKFILL_COVERAGE:.0%}"
|
||||
)
|
||||
|
||||
summary = {
|
||||
"backfill": staged.backfill,
|
||||
"issuers_fetched": staged.issuers_fetched,
|
||||
"issuers_with_rows": staged.issuers_with_rows,
|
||||
"snapshot_rows": len(staged.rows),
|
||||
"skipped_filings": len(staged.skipped_filings),
|
||||
"field_issues": len(staged.field_issues),
|
||||
"skipped_non_xbrl": len(staged.skipped_non_xbrl),
|
||||
"missing_xbrl": len(staged.missing_xbrl),
|
||||
"cik_updates": len(staged.resolved.cik_updates),
|
||||
}
|
||||
return ValidationResult(
|
||||
ok=not messages,
|
||||
summary=summary,
|
||||
source_max_date=self._latest_index_date,
|
||||
messages=messages,
|
||||
)
|
||||
|
||||
async def promote(self, db, staged: StagedFundamentals, run_id: int) -> dict[str, int]:
|
||||
inserted = 0
|
||||
discrepancies = 0
|
||||
if staged.rows:
|
||||
existing = await self._existing_by_accession(db, [r.accession for r in staged.rows])
|
||||
for row in staged.rows:
|
||||
old = existing.get(row.accession)
|
||||
if old is not None:
|
||||
if _differs(row, old):
|
||||
discrepancies += 1
|
||||
logger.warning(
|
||||
"sec_facts: accession %s reconstructed differently than "
|
||||
"stored (immutable — not overwriting)", row.accession
|
||||
)
|
||||
continue # ON CONFLICT DO NOTHING (below) leaves it untouched
|
||||
stmt = insert_for_session(db, FundamentalSnapshot).values(
|
||||
**_row_values(row, run_id)
|
||||
)
|
||||
stmt = stmt.on_conflict_do_nothing(index_elements=["accession"])
|
||||
await db.execute(stmt)
|
||||
inserted += 1
|
||||
|
||||
ticker_counts = await sec_universe.apply_ticker_updates(
|
||||
db, staged.resolved, staged.sic_updates
|
||||
)
|
||||
return {
|
||||
"inserted": inserted,
|
||||
"existing_unchanged": len(staged.rows) - inserted,
|
||||
"discrepancies": discrepancies,
|
||||
**ticker_counts,
|
||||
}
|
||||
|
||||
# -- helpers -----------------------------------------------------------
|
||||
|
||||
async def _last_processed_index_date(self, db) -> date | None:
|
||||
return (
|
||||
await db.execute(
|
||||
select(DataImportRun.source_max_date)
|
||||
.where(DataImportRun.source == SOURCE, DataImportRun.status == STATUS_PROMOTED)
|
||||
.order_by(DataImportRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
|
||||
async def _collect_index_rows(
|
||||
self, client: SecClient, last_processed: date, latest: date
|
||||
) -> list[dict[str, Any]]:
|
||||
tracked = set(self._resolved.cik_to_ticker_ids) if self._resolved else set()
|
||||
start = max(last_processed + timedelta(days=1), latest - timedelta(days=_MAX_INDEX_WALK_DAYS))
|
||||
if start > last_processed + timedelta(days=1):
|
||||
logger.warning("sec_facts: index gap > %d days; walking from %s", _MAX_INDEX_WALK_DAYS, start)
|
||||
rows: list[dict[str, Any]] = []
|
||||
day = start
|
||||
while day <= latest:
|
||||
for r in await client.daily_index(day):
|
||||
if r["form"] in _XBRL_FORMS and r["cik"] in tracked:
|
||||
rows.append(r)
|
||||
day += timedelta(days=1)
|
||||
return rows
|
||||
|
||||
async def _ciks_with_snapshots(self, db, ciks: set[int]) -> set[int]:
|
||||
if not ciks:
|
||||
return set()
|
||||
cik_strs = [cik10(c) for c in ciks]
|
||||
found = (
|
||||
await db.execute(
|
||||
select(FundamentalSnapshot.cik)
|
||||
.where(FundamentalSnapshot.cik.in_(cik_strs))
|
||||
.distinct()
|
||||
)
|
||||
).scalars().all()
|
||||
return {int(c) for c in found}
|
||||
|
||||
async def _existing_by_accession(self, db, accessions: list[str]) -> dict[str, FundamentalSnapshot]:
|
||||
if not accessions:
|
||||
return {}
|
||||
rows = (
|
||||
await db.execute(
|
||||
select(FundamentalSnapshot).where(FundamentalSnapshot.accession.in_(accessions))
|
||||
)
|
||||
).scalars().all()
|
||||
return {r.accession: r for r in rows}
|
||||
|
||||
|
||||
def _filing_meta(sub: dict[str, Any]) -> tuple[dict[str, FilingMeta], set[str]]:
|
||||
"""(xbrl_meta, nonxbrl_accessions) from a submissions payload. xbrl_meta only
|
||||
includes 10-K/10-Q(/A) filings that are XBRL and have full period metadata."""
|
||||
xbrl: dict[str, FilingMeta] = {}
|
||||
nonxbrl: set[str] = set()
|
||||
for f in sub.get("filings", []):
|
||||
if f["form"] not in _XBRL_FORMS:
|
||||
continue
|
||||
if not f.get("is_xbrl"):
|
||||
nonxbrl.add(f["accession"])
|
||||
continue
|
||||
if not (f.get("report_date") and f.get("filing_date") and f.get("acceptance_datetime")):
|
||||
continue
|
||||
xbrl[f["accession"]] = FilingMeta(
|
||||
report_date=date.fromisoformat(f["report_date"]),
|
||||
filing_date=date.fromisoformat(f["filing_date"]),
|
||||
accepted_at=_parse_dt(f["acceptance_datetime"]),
|
||||
form=f["form"],
|
||||
)
|
||||
return xbrl, nonxbrl
|
||||
|
||||
|
||||
def _parse_dt(value: str) -> datetime:
|
||||
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
|
||||
|
||||
def _row_values(row: SnapshotRow, run_id: int) -> dict[str, Any]:
|
||||
values = {col: getattr(row, col) for col in _SNAPSHOT_COLS}
|
||||
values["import_run_id"] = run_id
|
||||
values["created_at"] = _now()
|
||||
return values
|
||||
|
||||
|
||||
def _differs(row: SnapshotRow, old: FundamentalSnapshot) -> bool:
|
||||
return any(getattr(row, col) != getattr(old, col) for col in _DISCREPANCY_COLS)
|
||||
Reference in New Issue
Block a user