feat(sec): A3 slice 2b — SEC fundamentals importer (shadow ingestion)

SecFundamentalsImporter (SourceImporter, source=sec_facts): populates immutable
fundamental_snapshots from Company Facts and back-fills tickers.cik/sic, driven
by the EDGAR daily index. Shadow only. Guardrails per review:

- detect_revision caches the resolved universe + exact tracked index rows and
  composes the revision from them; stage consumes those same cached inputs
  (no index/universe refetch) so promoted data matches the computed revision.
- Resolution is read-only in stage (proposals only); ticker writes happen in
  promote via apply_ticker_updates.
- validate runs the index<->Company-Facts consistency gate before any write:
  a tracked XBRL index accession missing from Company Facts fails the run
  (they lag independently) so we retry, not record null. Non-XBRL amendments
  are skipped with a recorded reason. Backfill has a coverage floor.
- promote inserts ON CONFLICT (accession) DO NOTHING (immutable), reports
  differing existing accessions without mutating, and applies ticker updates in
  the same transaction.
- Full-history backfill on first run / for newly-added issuers (include_history);
  incremental fetch only for issuers that filed.

Parser: split parse result into skipped_filings vs field_issues (coverage must
not count field warnings); header notes the us-gaap shares fallback; added
companyfacts_accessions() for the gate.

Verified live end-to-end (AAPL + GOOGL backfill): 112 snapshots, cik/sic set,
GOOGL shares via us-gaap fallback, AAPL via dei. Tests: 6 importer (backfill,
incremental, consistency-gate fail, non-XBRL skip, read-only-on-failure,
conflict-discrepancy) + parser ParseResult updates.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-22 19:39:46 +02:00
co-authored by Claude Opus 4.8
parent 4754dbc17b
commit f7ce85a33e
4 changed files with 706 additions and 35 deletions
+23 -21
View File
@@ -66,9 +66,9 @@ def _by_accn(rows):
def test_parses_ytd_not_discrete_and_cover_date_shares():
rows, skips = parse_snapshots(COMPANYFACTS, FILINGS, {"A", "B"})
assert not skips
b = _by_accn(rows)["B"]
res = parse_snapshots(COMPANYFACTS, FILINGS, {"A", "B"})
assert not res.skipped_filings and not res.field_issues
b = _by_accn(res.rows)["B"]
assert (b.cik, b.fiscal_year, b.fiscal_period) == ("0000320193", 2026, "Q2")
assert b.period_end == date(2026, 3, 28)
@@ -88,8 +88,8 @@ def test_parses_ytd_not_discrete_and_cover_date_shares():
def test_q1_discrete_is_the_ytd():
rows, _ = parse_snapshots(COMPANYFACTS, FILINGS, {"A"})
a = _by_accn(rows)["A"]
res = parse_snapshots(COMPANYFACTS, FILINGS, {"A"})
a = _by_accn(res.rows)["A"]
assert a.fiscal_period == "Q1"
assert a.revenue == 143756 # Q1 YTD == Q1 discrete
assert a.period_start == date(2025, 9, 28)
@@ -103,15 +103,15 @@ def test_skips_filing_without_usable_period():
]}}}},
}
filings = {"X": FilingMeta(date(2026, 3, 28), date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
rows, skips = parse_snapshots(cf, filings, {"X"})
assert rows == []
assert skips == [{"accession": "X", "reason": "no usable period identity"}]
res = parse_snapshots(cf, filings, {"X"})
assert res.rows == []
assert res.skipped_filings == [{"accession": "X", "reason": "no usable period identity"}]
def test_missing_accession_is_skipped():
rows, skips = parse_snapshots(COMPANYFACTS, FILINGS, {"NOPE"})
assert rows == []
assert skips == [{"accession": "NOPE", "reason": "no facts or filing metadata"}]
res = parse_snapshots(COMPANYFACTS, FILINGS, {"NOPE"})
assert res.rows == []
assert res.skipped_filings == [{"accession": "NOPE", "reason": "no facts or filing metadata"}]
def test_debt_prefers_aggregate_over_parts():
@@ -186,8 +186,9 @@ def test_conflicting_context_skips_row():
"NetIncomeLoss": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 2, "B", fy=2025, fp="Q3")]}},
}}}
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
rows, skips = parse_snapshots(cf, filings, {"B"})
assert rows == [] and skips == [{"accession": "B", "reason": "no usable period identity"}]
res = parse_snapshots(cf, filings, {"B"})
assert res.rows == []
assert res.skipped_filings == [{"accession": "B", "reason": "no usable period identity"}]
def test_foreign_taxonomy_and_malformed_facts_ignored():
@@ -206,10 +207,10 @@ def test_foreign_taxonomy_and_malformed_facts_ignored():
]}}},
}}
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
rows, _ = parse_snapshots(cf, filings, {"B"})
assert rows[0].revenue == 500 # us-gaap Revenues, not the acme concept
assert rows[0].net_income is None # val None ignored
assert rows[0].operating_income is None # NaN ignored
res = parse_snapshots(cf, filings, {"B"})
assert res.rows[0].revenue == 500 # us-gaap Revenues, not the acme concept
assert res.rows[0].net_income is None # val None ignored
assert res.rows[0].operating_income is None # NaN ignored
def test_ambiguous_shares_produces_row_plus_note():
@@ -221,9 +222,10 @@ def test_ambiguous_shares_produces_row_plus_note():
]}}},
}}
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
rows, skips = parse_snapshots(cf, filings, {"B"})
assert len(rows) == 1 and rows[0].shares_outstanding is None # row kept, shares null
assert {"accession": "B", "reason": "ambiguous shares outstanding"} in skips
res = parse_snapshots(cf, filings, {"B"})
assert len(res.rows) == 1 and res.rows[0].shares_outstanding is None # row kept, shares null
assert res.field_issues == [{"accession": "B", "reason": "ambiguous shares outstanding"}]
assert res.skipped_filings == [] # a field issue is NOT a skipped filing
# Opt-in live check against real Apple companyfacts. Skips unless SEC_LIVE=1 and a
@@ -252,7 +254,7 @@ async def test_live_apple_parse_invariants():
for f in sub["filings"]
if f["report_date"] and f["filing_date"] and f["acceptance_datetime"]
}
rows, _ = parse_snapshots(cf, filings, set(filings))
rows = parse_snapshots(cf, filings, set(filings)).rows
assert len(rows) > 20
assert all(r.period_end and r.fiscal_year and r.fiscal_period for r in rows)
# YTD revenue is non-decreasing within a fiscal year