feat(sec): A3 slice 2b — SEC fundamentals importer (shadow ingestion)
SecFundamentalsImporter (SourceImporter, source=sec_facts): populates immutable fundamental_snapshots from Company Facts and back-fills tickers.cik/sic, driven by the EDGAR daily index. Shadow only. Guardrails per review: - detect_revision caches the resolved universe + exact tracked index rows and composes the revision from them; stage consumes those same cached inputs (no index/universe refetch) so promoted data matches the computed revision. - Resolution is read-only in stage (proposals only); ticker writes happen in promote via apply_ticker_updates. - validate runs the index<->Company-Facts consistency gate before any write: a tracked XBRL index accession missing from Company Facts fails the run (they lag independently) so we retry, not record null. Non-XBRL amendments are skipped with a recorded reason. Backfill has a coverage floor. - promote inserts ON CONFLICT (accession) DO NOTHING (immutable), reports differing existing accessions without mutating, and applies ticker updates in the same transaction. - Full-history backfill on first run / for newly-added issuers (include_history); incremental fetch only for issuers that filed. Parser: split parse result into skipped_filings vs field_issues (coverage must not count field warnings); header notes the us-gaap shares fallback; added companyfacts_accessions() for the gate. Verified live end-to-end (AAPL + GOOGL backfill): 112 snapshots, cik/sic set, GOOGL shares via us-gaap fallback, AAPL via dei. Tests: 6 importer (backfill, incremental, consistency-gate fail, non-XBRL skip, read-only-on-failure, conflict-discrepancy) + parser ParseResult updates. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
@@ -66,9 +66,9 @@ def _by_accn(rows):
|
||||
|
||||
|
||||
def test_parses_ytd_not_discrete_and_cover_date_shares():
|
||||
rows, skips = parse_snapshots(COMPANYFACTS, FILINGS, {"A", "B"})
|
||||
assert not skips
|
||||
b = _by_accn(rows)["B"]
|
||||
res = parse_snapshots(COMPANYFACTS, FILINGS, {"A", "B"})
|
||||
assert not res.skipped_filings and not res.field_issues
|
||||
b = _by_accn(res.rows)["B"]
|
||||
|
||||
assert (b.cik, b.fiscal_year, b.fiscal_period) == ("0000320193", 2026, "Q2")
|
||||
assert b.period_end == date(2026, 3, 28)
|
||||
@@ -88,8 +88,8 @@ def test_parses_ytd_not_discrete_and_cover_date_shares():
|
||||
|
||||
|
||||
def test_q1_discrete_is_the_ytd():
|
||||
rows, _ = parse_snapshots(COMPANYFACTS, FILINGS, {"A"})
|
||||
a = _by_accn(rows)["A"]
|
||||
res = parse_snapshots(COMPANYFACTS, FILINGS, {"A"})
|
||||
a = _by_accn(res.rows)["A"]
|
||||
assert a.fiscal_period == "Q1"
|
||||
assert a.revenue == 143756 # Q1 YTD == Q1 discrete
|
||||
assert a.period_start == date(2025, 9, 28)
|
||||
@@ -103,15 +103,15 @@ def test_skips_filing_without_usable_period():
|
||||
]}}}},
|
||||
}
|
||||
filings = {"X": FilingMeta(date(2026, 3, 28), date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
|
||||
rows, skips = parse_snapshots(cf, filings, {"X"})
|
||||
assert rows == []
|
||||
assert skips == [{"accession": "X", "reason": "no usable period identity"}]
|
||||
res = parse_snapshots(cf, filings, {"X"})
|
||||
assert res.rows == []
|
||||
assert res.skipped_filings == [{"accession": "X", "reason": "no usable period identity"}]
|
||||
|
||||
|
||||
def test_missing_accession_is_skipped():
|
||||
rows, skips = parse_snapshots(COMPANYFACTS, FILINGS, {"NOPE"})
|
||||
assert rows == []
|
||||
assert skips == [{"accession": "NOPE", "reason": "no facts or filing metadata"}]
|
||||
res = parse_snapshots(COMPANYFACTS, FILINGS, {"NOPE"})
|
||||
assert res.rows == []
|
||||
assert res.skipped_filings == [{"accession": "NOPE", "reason": "no facts or filing metadata"}]
|
||||
|
||||
|
||||
def test_debt_prefers_aggregate_over_parts():
|
||||
@@ -186,8 +186,9 @@ def test_conflicting_context_skips_row():
|
||||
"NetIncomeLoss": {"units": {"USD": [_dur("2025-09-28", "2026-03-28", 2, "B", fy=2025, fp="Q3")]}},
|
||||
}}}
|
||||
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
|
||||
rows, skips = parse_snapshots(cf, filings, {"B"})
|
||||
assert rows == [] and skips == [{"accession": "B", "reason": "no usable period identity"}]
|
||||
res = parse_snapshots(cf, filings, {"B"})
|
||||
assert res.rows == []
|
||||
assert res.skipped_filings == [{"accession": "B", "reason": "no usable period identity"}]
|
||||
|
||||
|
||||
def test_foreign_taxonomy_and_malformed_facts_ignored():
|
||||
@@ -206,10 +207,10 @@ def test_foreign_taxonomy_and_malformed_facts_ignored():
|
||||
]}}},
|
||||
}}
|
||||
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
|
||||
rows, _ = parse_snapshots(cf, filings, {"B"})
|
||||
assert rows[0].revenue == 500 # us-gaap Revenues, not the acme concept
|
||||
assert rows[0].net_income is None # val None ignored
|
||||
assert rows[0].operating_income is None # NaN ignored
|
||||
res = parse_snapshots(cf, filings, {"B"})
|
||||
assert res.rows[0].revenue == 500 # us-gaap Revenues, not the acme concept
|
||||
assert res.rows[0].net_income is None # val None ignored
|
||||
assert res.rows[0].operating_income is None # NaN ignored
|
||||
|
||||
|
||||
def test_ambiguous_shares_produces_row_plus_note():
|
||||
@@ -221,9 +222,10 @@ def test_ambiguous_shares_produces_row_plus_note():
|
||||
]}}},
|
||||
}}
|
||||
filings = {"B": FilingMeta(RD, date(2026, 5, 1), datetime(2026, 5, 1, tzinfo=UTC), "10-Q")}
|
||||
rows, skips = parse_snapshots(cf, filings, {"B"})
|
||||
assert len(rows) == 1 and rows[0].shares_outstanding is None # row kept, shares null
|
||||
assert {"accession": "B", "reason": "ambiguous shares outstanding"} in skips
|
||||
res = parse_snapshots(cf, filings, {"B"})
|
||||
assert len(res.rows) == 1 and res.rows[0].shares_outstanding is None # row kept, shares null
|
||||
assert res.field_issues == [{"accession": "B", "reason": "ambiguous shares outstanding"}]
|
||||
assert res.skipped_filings == [] # a field issue is NOT a skipped filing
|
||||
|
||||
|
||||
# Opt-in live check against real Apple companyfacts. Skips unless SEC_LIVE=1 and a
|
||||
@@ -252,7 +254,7 @@ async def test_live_apple_parse_invariants():
|
||||
for f in sub["filings"]
|
||||
if f["report_date"] and f["filing_date"] and f["acceptance_datetime"]
|
||||
}
|
||||
rows, _ = parse_snapshots(cf, filings, set(filings))
|
||||
rows = parse_snapshots(cf, filings, set(filings)).rows
|
||||
assert len(rows) > 20
|
||||
assert all(r.period_end and r.fiscal_year and r.fiscal_period for r in rows)
|
||||
# YTD revenue is non-decreasing within a fiscal year
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
"""Integration tests for the SEC fundamentals importer, driven through the real
|
||||
import framework with a fake SEC client (no network)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
|
||||
from app.database import Base
|
||||
import app.models # noqa: F401
|
||||
from app.models.fundamental_snapshot import FundamentalSnapshot
|
||||
from app.models.ticker import Ticker
|
||||
from app.services.data_import import STATUS_FAILED, STATUS_PROMOTED, run_import
|
||||
from app.services.sec_fundamentals_importer import SecFundamentalsImporter
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def engine():
|
||||
fd, path = tempfile.mkstemp(suffix=".db")
|
||||
os.close(fd)
|
||||
eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
|
||||
async with eng.begin() as conn:
|
||||
await conn.run_sync(Base.metadata.create_all)
|
||||
try:
|
||||
yield eng
|
||||
finally:
|
||||
await eng.dispose()
|
||||
try:
|
||||
os.unlink(path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _factory(engine):
|
||||
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
||||
|
||||
|
||||
# --- fixture SEC data (AAPL, cik 320193) -----------------------------------
|
||||
|
||||
def _rev(start, end, val, fy, fp, accn):
|
||||
return {"start": start, "end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
|
||||
|
||||
|
||||
def _shares(end, val, accn, fy, fp):
|
||||
return {"end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
|
||||
|
||||
|
||||
def _companyfacts(rev_facts, share_facts):
|
||||
return {
|
||||
"cik": 320193,
|
||||
"facts": {
|
||||
"us-gaap": {"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": rev_facts}}},
|
||||
"dei": {"EntityCommonStockSharesOutstanding": {"units": {"shares": share_facts}}},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _filing(accn, form, report, filed, accepted, is_xbrl=True):
|
||||
return {"accession": accn, "form": form, "report_date": report, "filing_date": filed,
|
||||
"acceptance_datetime": accepted, "is_xbrl": is_xbrl}
|
||||
|
||||
|
||||
CF_K = _rev("2024-09-29", "2025-09-27", 416161, 2025, "FY", "K")
|
||||
CF_Q1 = _rev("2025-09-28", "2025-12-27", 143756, 2026, "Q1", "Q")
|
||||
SH_K = _shares("2025-10-17", 14776, "K", 2025, "FY")
|
||||
SH_Q1 = _shares("2026-01-16", 14681, "Q", 2026, "Q1")
|
||||
|
||||
SUB_FILINGS = [
|
||||
_filing("K", "10-K", "2025-09-27", "2025-10-31", "2025-10-31T10:01:26.000Z"),
|
||||
_filing("Q", "10-Q", "2025-12-27", "2026-01-30", "2026-01-30T11:01:00.000Z"),
|
||||
]
|
||||
|
||||
|
||||
def _submissions(filings):
|
||||
return {"cik": 320193, "sic": "3571", "sic_description": "Electronic Computers",
|
||||
"fiscal_year_end": "0926", "tickers": ["AAPL"], "filings": filings}
|
||||
|
||||
|
||||
class FakeSecClient:
|
||||
def __init__(self, *, tickers, companyfacts, submissions, latest_index, daily=None):
|
||||
self._tickers = tickers
|
||||
self._cf = companyfacts
|
||||
self._sub = submissions
|
||||
self._latest = latest_index
|
||||
self._daily = daily or {}
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *a):
|
||||
return False
|
||||
|
||||
async def company_tickers(self):
|
||||
return dict(self._tickers)
|
||||
|
||||
async def latest_index_date(self, today=None):
|
||||
return self._latest
|
||||
|
||||
async def daily_index(self, day):
|
||||
return list(self._daily.get(day, []))
|
||||
|
||||
async def companyfacts(self, cik):
|
||||
return self._cf[int(cik)]
|
||||
|
||||
async def submissions(self, cik, *, include_history=False):
|
||||
return self._sub[int(cik)]
|
||||
|
||||
|
||||
def _importer(client, today=date(2026, 2, 1)):
|
||||
return SecFundamentalsImporter(client_factory=lambda: client, today=today)
|
||||
|
||||
|
||||
async def _seed(factory, symbols):
|
||||
async with factory() as s:
|
||||
for sym in symbols:
|
||||
s.add(Ticker(symbol=sym))
|
||||
await s.commit()
|
||||
|
||||
|
||||
async def _count(factory, model):
|
||||
async with factory() as s:
|
||||
return (await s.execute(select(func.count()).select_from(model))).scalar_one()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_backfill_inserts_snapshots_and_ticker_meta(engine):
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL"])
|
||||
client = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
||||
submissions={320193: _submissions(SUB_FILINGS)},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
run = await run_import(_importer(client), engine=engine)
|
||||
|
||||
assert run.status == STATUS_PROMOTED
|
||||
assert run.source_max_date == date(2026, 1, 31)
|
||||
assert await _count(factory, FundamentalSnapshot) == 2
|
||||
|
||||
async with factory() as s:
|
||||
t = (await s.execute(select(Ticker))).scalar_one()
|
||||
assert t.cik == "0000320193" and t.sic == "3571"
|
||||
snaps = (await s.execute(select(FundamentalSnapshot))).scalars().all()
|
||||
assert {x.fiscal_period for x in snaps} == {"FY", "Q1"}
|
||||
assert all(x.import_run_id == run.id for x in snaps)
|
||||
fy = next(x for x in snaps if x.fiscal_period == "FY")
|
||||
assert fy.revenue == 416161 and fy.shares_outstanding == 14776
|
||||
|
||||
|
||||
async def test_incremental_adds_only_new_filing(engine):
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL"])
|
||||
backfill = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
||||
submissions={320193: _submissions(SUB_FILINGS)},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
await run_import(_importer(backfill), engine=engine)
|
||||
assert await _count(factory, FundamentalSnapshot) == 2
|
||||
|
||||
# A new Q2 10-Q appears in the daily index and Company Facts.
|
||||
cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A")
|
||||
sh_q2 = _shares("2026-04-17", 14687, "Q2A", 2026, "Q2")
|
||||
incr = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])},
|
||||
submissions={320193: _submissions(SUB_FILINGS + [
|
||||
_filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
|
||||
latest_index=date(2026, 5, 2),
|
||||
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "Q2A"}]},
|
||||
)
|
||||
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
||||
|
||||
assert run.status == STATUS_PROMOTED
|
||||
assert await _count(factory, FundamentalSnapshot) == 3 # only Q2A added
|
||||
async with factory() as s:
|
||||
q2 = (await s.execute(
|
||||
select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A")
|
||||
)).scalar_one()
|
||||
assert q2.fiscal_period == "Q2" and q2.revenue == 254940
|
||||
|
||||
|
||||
async def test_consistency_gate_fails_when_facts_lag_index(engine):
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL"])
|
||||
backfill = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
||||
submissions={320193: _submissions(SUB_FILINGS)},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
await run_import(_importer(backfill), engine=engine)
|
||||
|
||||
# Index + submissions list an XBRL filing "GHOST" that Company Facts lacks.
|
||||
incr = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # no GHOST
|
||||
submissions={320193: _submissions(SUB_FILINGS + [
|
||||
_filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
|
||||
latest_index=date(2026, 5, 2),
|
||||
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]},
|
||||
)
|
||||
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
||||
|
||||
assert run.status == STATUS_FAILED
|
||||
assert "Company Facts" in (run.error_details or "")
|
||||
assert await _count(factory, FundamentalSnapshot) == 2 # nothing new written
|
||||
|
||||
|
||||
async def test_non_xbrl_amendment_skipped_not_failed(engine):
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL"])
|
||||
backfill = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
||||
submissions={320193: _submissions(SUB_FILINGS)},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
await run_import(_importer(backfill), engine=engine)
|
||||
|
||||
incr = FakeSecClient(
|
||||
tickers={"AAPL": 320193},
|
||||
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
||||
submissions={320193: _submissions(SUB_FILINGS + [
|
||||
_filing("AMD", "10-K/A", "2025-09-27", "2026-05-01", "2026-05-01T10:01:00.000Z", is_xbrl=False)])},
|
||||
latest_index=date(2026, 5, 2),
|
||||
daily={date(2026, 5, 1): [{"form": "10-K/A", "cik": 320193, "accession": "AMD"}]},
|
||||
)
|
||||
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
||||
|
||||
assert run.status == STATUS_PROMOTED # non-XBRL amendment is skipped, not a failure
|
||||
assert "skipped_non_xbrl" in (run.validation_json or "")
|
||||
assert await _count(factory, FundamentalSnapshot) == 2
|
||||
|
||||
|
||||
async def test_failed_backfill_leaves_tickers_unwritten(engine):
|
||||
factory = _factory(engine)
|
||||
await _seed(factory, ["AAPL", "MSFT", "NVDA"]) # 3 resolve, only AAPL yields rows
|
||||
client = FakeSecClient(
|
||||
tickers={"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810},
|
||||
companyfacts={
|
||||
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
||||
789019: {"cik": 789019, "facts": {}}, # no facts -> no rows
|
||||
1045810: {"cik": 1045810, "facts": {}},
|
||||
},
|
||||
submissions={
|
||||
320193: _submissions(SUB_FILINGS),
|
||||
789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
|
||||
1045810: {"cik": 1045810, "sic": "3674", "sic_description": "y", "filings": []},
|
||||
},
|
||||
latest_index=date(2026, 1, 31),
|
||||
)
|
||||
run = await run_import(_importer(client), engine=engine)
|
||||
|
||||
assert run.status == STATUS_FAILED # coverage 1/3 < 50%
|
||||
assert "coverage" in (run.error_details or "")
|
||||
assert await _count(factory, FundamentalSnapshot) == 0
|
||||
# read-only resolution: no ticker cik/sic written on a failed run
|
||||
async with factory() as s:
|
||||
assert all(t.cik is None and t.sic is None for t in (await s.execute(select(Ticker))).scalars())
|
||||
|
||||
|
||||
async def test_promote_conflict_reports_discrepancy_without_mutation(engine):
|
||||
from app.services.sec_facts_parser import SnapshotRow
|
||||
from app.services.sec_fundamentals_importer import StagedFundamentals
|
||||
from app.services.sec_universe import ResolvedUniverse
|
||||
|
||||
factory = _factory(engine)
|
||||
utc = timezone.utc
|
||||
async with factory() as s: # pre-existing immutable snapshot K (revenue 100, run 1)
|
||||
s.add(FundamentalSnapshot(
|
||||
cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31),
|
||||
accepted_at=datetime(2025, 10, 31, tzinfo=utc), period_end=date(2025, 9, 27),
|
||||
fiscal_year=2025, fiscal_period="FY", revenue=100.0, import_run_id=1,
|
||||
created_at=datetime(2025, 10, 31, tzinfo=utc)))
|
||||
await s.commit()
|
||||
|
||||
k_diff = SnapshotRow(cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31),
|
||||
accepted_at=datetime(2025, 10, 31, tzinfo=utc), period_end=date(2025, 9, 27),
|
||||
fiscal_year=2025, fiscal_period="FY", revenue=999.0) # differs
|
||||
n_new = SnapshotRow(cik="0000320193", accession="N", form="10-Q", filed_date=date(2026, 1, 30),
|
||||
accepted_at=datetime(2026, 1, 30, tzinfo=utc), period_end=date(2025, 12, 27),
|
||||
fiscal_year=2026, fiscal_period="Q1", revenue=143.0)
|
||||
staged = StagedFundamentals(resolved=ResolvedUniverse(), rows=[k_diff, n_new])
|
||||
|
||||
imp = SecFundamentalsImporter(client_factory=lambda: None)
|
||||
async with factory() as s:
|
||||
counts = await imp.promote(s, staged, run_id=2)
|
||||
await s.commit()
|
||||
|
||||
assert counts["inserted"] == 1 and counts["discrepancies"] == 1
|
||||
async with factory() as s:
|
||||
k = (await s.execute(select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "K"))).scalar_one()
|
||||
assert k.revenue == 100.0 and k.import_run_id == 1 # immutable — not overwritten
|
||||
assert (await s.execute(select(func.count()).select_from(FundamentalSnapshot))).scalar_one() == 2
|
||||
Reference in New Issue
Block a user