feat(sec): A3 slice 2b — SEC fundamentals importer (shadow ingestion)
SecFundamentalsImporter (SourceImporter, source=sec_facts): populates immutable fundamental_snapshots from Company Facts and back-fills tickers.cik/sic, driven by the EDGAR daily index. Shadow only. Guardrails per review: - detect_revision caches the resolved universe + exact tracked index rows and composes the revision from them; stage consumes those same cached inputs (no index/universe refetch) so promoted data matches the computed revision. - Resolution is read-only in stage (proposals only); ticker writes happen in promote via apply_ticker_updates. - validate runs the index<->Company-Facts consistency gate before any write: a tracked XBRL index accession missing from Company Facts fails the run (they lag independently) so we retry, not record null. Non-XBRL amendments are skipped with a recorded reason. Backfill has a coverage floor. - promote inserts ON CONFLICT (accession) DO NOTHING (immutable), reports differing existing accessions without mutating, and applies ticker updates in the same transaction. - Full-history backfill on first run / for newly-added issuers (include_history); incremental fetch only for issuers that filed. Parser: split parse result into skipped_filings vs field_issues (coverage must not count field warnings); header notes the us-gaap shares fallback; added companyfacts_accessions() for the gate. Verified live end-to-end (AAPL + GOOGL backfill): 112 snapshots, cik/sic set, GOOGL shares via us-gaap fallback, AAPL via dei. Tests: 6 importer (backfill, incremental, consistency-gate fail, non-XBRL skip, read-only-on-failure, conflict-discrepancy) + parser ParseResult updates. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -11,17 +11,24 @@ The load-bearing rules (design Decision 2 + review):
|
||||
- Duration facts are stored as **cumulative YTD**: pick the fact whose span
|
||||
matches the fiscal-period-to-date length (Q1≈3mo … FY≈12mo) within tolerance.
|
||||
If no YTD-length fact exists, store null — never a discrete masquerading as YTD.
|
||||
- Balance-sheet instants are taken at `end == reportDate`; `shares_outstanding`
|
||||
is the cover-page `dei` fact whose own `end` (cover date) is stored separately.
|
||||
- Balance-sheet instants are taken at `end == reportDate`. `shares_outstanding`
|
||||
is a single consolidated value: the cover-page `dei` fact (its own cover-date
|
||||
`end` stored separately) if present, else `us-gaap:CommonStockSharesOutstanding`
|
||||
at period end (e.g. Alphabet has no `dei` fact) — never a class sum or the
|
||||
weighted-average/diluted count.
|
||||
- Cash and debt composites are aggregate-first and mutually exclusive (each
|
||||
source tag counted at most once).
|
||||
|
||||
`parse_snapshots` separates `skipped_filings` (no usable row produced) from
|
||||
`field_issues` (a row was produced but a field is null/ambiguous) — callers must
|
||||
not treat field issues as missing coverage.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, datetime
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
@@ -107,34 +114,49 @@ class FilingMeta:
|
||||
form: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParseResult:
|
||||
rows: list[SnapshotRow] = field(default_factory=list)
|
||||
# accessions for which NO row was produced (no facts / no usable period).
|
||||
skipped_filings: list[dict[str, str]] = field(default_factory=list)
|
||||
# accessions with a row but a field-level warning (e.g. ambiguous shares).
|
||||
field_issues: list[dict[str, str]] = field(default_factory=list)
|
||||
|
||||
|
||||
def parse_snapshots(
|
||||
companyfacts: dict[str, Any],
|
||||
filings: dict[str, FilingMeta],
|
||||
accessions: set[str],
|
||||
) -> tuple[list[SnapshotRow], list[dict[str, str]]]:
|
||||
) -> ParseResult:
|
||||
"""Build snapshot rows for ``accessions`` (those with facts + filing meta).
|
||||
|
||||
Returns (rows, skips) where each skip is {accession, reason} for filings
|
||||
with no usable period identity — the caller counts these in validation_json.
|
||||
``skipped_filings`` = no row produced (missing facts/meta or no usable period
|
||||
identity); ``field_issues`` = a row was produced but a field is null/ambiguous.
|
||||
Callers must not use field issues as failed-row coverage.
|
||||
"""
|
||||
cik = f"{int(companyfacts['cik']):010d}"
|
||||
by_accn = _index_by_accession(companyfacts)
|
||||
rows: list[SnapshotRow] = []
|
||||
skips: list[dict[str, str]] = []
|
||||
result = ParseResult()
|
||||
for accn in accessions:
|
||||
meta = filings.get(accn)
|
||||
facts = by_accn.get(accn)
|
||||
if meta is None or not facts:
|
||||
skips.append({"accession": accn, "reason": "no facts or filing metadata"})
|
||||
result.skipped_filings.append({"accession": accn, "reason": "no facts or filing metadata"})
|
||||
continue
|
||||
row, note = _parse_one(cik, accn, facts, meta)
|
||||
if row is None:
|
||||
skips.append({"accession": accn, "reason": note or "unparseable"})
|
||||
result.skipped_filings.append({"accession": accn, "reason": note or "unparseable"})
|
||||
continue
|
||||
rows.append(row)
|
||||
if note: # row produced, but a field-level issue to count in validation
|
||||
skips.append({"accession": accn, "reason": note})
|
||||
return rows, skips
|
||||
result.rows.append(row)
|
||||
if note:
|
||||
result.field_issues.append({"accession": accn, "reason": note})
|
||||
return result
|
||||
|
||||
|
||||
def companyfacts_accessions(companyfacts: dict[str, Any]) -> set[str]:
|
||||
"""Every accession that appears anywhere in a companyfacts payload — used by
|
||||
the importer's index↔Company-Facts consistency gate."""
|
||||
return set(_index_by_accession(companyfacts).keys())
|
||||
|
||||
|
||||
def _index_by_accession(companyfacts: dict[str, Any]) -> dict[str, list[Fact]]:
|
||||
|
||||
Reference in New Issue
Block a user