feat(sec): A3 slice 2b — SEC fundamentals importer (shadow ingestion)
SecFundamentalsImporter (SourceImporter, source=sec_facts): populates immutable fundamental_snapshots from Company Facts and back-fills tickers.cik/sic, driven by the EDGAR daily index. Shadow only. Guardrails per review: - detect_revision caches the resolved universe + exact tracked index rows and composes the revision from them; stage consumes those same cached inputs (no index/universe refetch) so promoted data matches the computed revision. - Resolution is read-only in stage (proposals only); ticker writes happen in promote via apply_ticker_updates. - validate runs the index<->Company-Facts consistency gate before any write: a tracked XBRL index accession missing from Company Facts fails the run (they lag independently) so we retry, not record null. Non-XBRL amendments are skipped with a recorded reason. Backfill has a coverage floor. - promote inserts ON CONFLICT (accession) DO NOTHING (immutable), reports differing existing accessions without mutating, and applies ticker updates in the same transaction. - Full-history backfill on first run / for newly-added issuers (include_history); incremental fetch only for issuers that filed. Parser: split parse result into skipped_filings vs field_issues (coverage must not count field warnings); header notes the us-gaap shares fallback; added companyfacts_accessions() for the gate. Verified live end-to-end (AAPL + GOOGL backfill): 112 snapshots, cik/sic set, GOOGL shares via us-gaap fallback, AAPL via dei. Tests: 6 importer (backfill, incremental, consistency-gate fail, non-XBRL skip, read-only-on-failure, conflict-discrepancy) + parser ParseResult updates. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -66,9 +66,9 @@ def _by_accn(rows):
|
||||
|
||||
|
||||
def test_parses_ytd_not_discrete_and_cover_date_shares():
|
||||
rows, skips = parse_snapshots(COMPANYFACTS, FILINGS, {"A", "B"})
|
||||
assert not skips
|
||||
b = _by_accn(rows)["B"]
|
||||
res = parse_snapshots(COMPANYFACTS, FILINGS, {"A", "B"})
|
||||
assert not res.skipped_filings and not res.field_issues
|
||||
b = _by_accn(res.rows)["B"]
|
||||
|
||||
assert (b.cik, b.fiscal_year, b.fiscal_period) == ("0000320193", 2026, "Q2")
|
||||
assert b.period_end == date(2026, 3, 28)
|
||||
@@ -88,8 +88,8 @@ def test_parses_ytd_not_discrete_and_cover_date_shares():
|
||||
|
||||
|
||||
def test_q1_discrete_is_the_ytd():
|
||||
rows, _ = parse_snapshots(COMPANYFACTS, FILINGS, {"A"})
|
||||
a = _by_accn(rows)["A"]
|
||||
res = parse_snapshots(COMPANYFACTS, FILINGS, {"A"})
|
||||
a = _by_accn(res.rows)["A"]
|
||||
assert a.fiscal_period == "Q1"
|
||||
assert a.revenue == 143756 # Q1 YTD == Q1 discrete
|
||||
assert a.period_start == date(2025, 9, 28)
|
||||
@@ -103,15 +103,15 @@ def test_skips_filing_without_usable_period():
|
||||
]}}}},
|
||||
}
|
||||
filings = {"X": FilingMeta(date(2026, 3, 28), date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
|
||||
rows, skips = parse_snapshots(cf, filings, {"X"})
|
||||
assert rows == []
|
||||
assert skips == [{"accession": "X", "reason": "no usable period identity"}]
|
||||
res = parse_snapshots(cf, filings, {"X"})
|
||||
assert res.rows == []
|
||||
assert res.skipped_filings == [{"accession": "X", "reason": "no usable period identity"}]
|
||||
|
||||
|
||||
def test_missing_accession_is_skipped():
|
||||
rows, skips = parse_snapshots(COMPANYFACTS, FILINGS, {"NOPE"})
|
||||
assert rows == []
|
||||
assert skips == [{"accession": "NOPE", "reason": "no facts or filing metadata"}]
|
||||
res = parse_snapshots(COMPANYFACTS, FILINGS, {"NOPE"})
|
||||
assert res.rows == []
|
||||
assert res.skipped_filings == [{"accession": "NOPE", "reason": "no facts or filing metadata"}]
|
||||
|
||||
|
||||
def test_debt_prefers_aggregate_over_parts():
|
||||
@@ -186,8 +186,9 @@ def test_conflicting_context_skips_row():
|
||||
"NetIncomeLoss": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 2, "B", fy=2025, fp="Q3")]}},
|
||||
}}}
|
||||
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
|
||||
rows, skips = parse_snapshots(cf, filings, {"B"})
|
||||
assert rows == [] and skips == [{"accession": "B", "reason": "no usable period identity"}]
|
||||
res = parse_snapshots(cf, filings, {"B"})
|
||||
assert res.rows == []
|
||||
assert res.skipped_filings == [{"accession": "B", "reason": "no usable period identity"}]
|
||||
|
||||
|
||||
def test_foreign_taxonomy_and_malformed_facts_ignored():
|
||||
@@ -206,10 +207,10 @@ def test_foreign_taxonomy_and_malformed_facts_ignored():
|
||||
]}}},
|
||||
}}
|
||||
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
|
||||
rows, _ = parse_snapshots(cf, filings, {"B"})
|
||||
assert rows[0].revenue == 500 # us-gaap Revenues, not the acme concept
|
||||
assert rows[0].net_income is None # val None ignored
|
||||
assert rows[0].operating_income is None # NaN ignored
|
||||
res = parse_snapshots(cf, filings, {"B"})
|
||||
assert res.rows[0].revenue == 500 # us-gaap Revenues, not the acme concept
|
||||
assert res.rows[0].net_income is None # val None ignored
|
||||
assert res.rows[0].operating_income is None # NaN ignored
|
||||
|
||||
|
||||
def test_ambiguous_shares_produces_row_plus_note():
|
||||
@@ -221,9 +222,10 @@ def test_ambiguous_shares_produces_row_plus_note():
|
||||
]}}},
|
||||
}}
|
||||
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
|
||||
rows, skips = parse_snapshots(cf, filings, {"B"})
|
||||
assert len(rows) == 1 and rows[0].shares_outstanding is None # row kept, shares null
|
||||
assert {"accession": "B", "reason": "ambiguous shares outstanding"} in skips
|
||||
res = parse_snapshots(cf, filings, {"B"})
|
||||
assert len(res.rows) == 1 and res.rows[0].shares_outstanding is None # row kept, shares null
|
||||
assert res.field_issues == [{"accession": "B", "reason": "ambiguous shares outstanding"}]
|
||||
assert res.skipped_filings == [] # a field issue is NOT a skipped filing
|
||||
|
||||
|
||||
# Opt-in live check against real Apple companyfacts. Skips unless SEC_LIVE=1 and a
|
||||
@@ -252,7 +254,7 @@ async def test_live_apple_parse_invariants():
|
||||
for f in sub["filings"]
|
||||
if f["report_date"] and f["filing_date"] and f["acceptance_datetime"]
|
||||
}
|
||||
rows, _ = parse_snapshots(cf, filings, set(filings))
|
||||
rows = parse_snapshots(cf, filings, set(filings)).rows
|
||||
assert len(rows) > 20
|
||||
assert all(r.period_end and r.fiscal_year and r.fiscal_period for r in rows)
|
||||
# YTD revenue is non-decreasing within a fiscal year
|
||||
|
||||
Reference in New Issue
Block a user