feat(sec): A3 slice 2b — SEC fundamentals importer (shadow ingestion)

SecFundamentalsImporter (SourceImporter, source=sec_facts): populates immutable fundamental_snapshots from Company Facts and back-fills tickers.cik/sic, driven by the EDGAR daily index. Shadow only. Guardrails per review: - detect_revision caches the resolved universe + exact tracked index rows and composes the revision from them; stage consumes those same cached inputs (no index/universe refetch) so promoted data matches the computed revision. - Resolution is read-only in stage (proposals only); ticker writes happen in promote via apply_ticker_updates. - validate runs the index<->Company-Facts consistency gate before any write: a tracked XBRL index accession missing from Company Facts fails the run (they lag independently) so we retry, not record null. Non-XBRL amendments are skipped with a recorded reason. Backfill has a coverage floor. - promote inserts ON CONFLICT (accession) DO NOTHING (immutable), reports differing existing accessions without mutating, and applies ticker updates in the same transaction. - Full-history backfill on first run / for newly-added issuers (include_history); incremental fetch only for issuers that filed. Parser: split parse result into skipped_filings vs field_issues (coverage must not count field warnings); header notes the us-gaap shares fallback; added companyfacts_accessions() for the gate. Verified live end-to-end (AAPL + GOOGL backfill): 112 snapshots, cik/sic set, GOOGL shares via us-gaap fallback, AAPL via dei. Tests: 6 importer (backfill, incremental, consistency-gate fail, non-XBRL skip, read-only-on-failure, conflict-discrepancy) + parser ParseResult updates. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-22 19:39:46 +02:00
parent 4754dbc17b
commit f7ce85a33e
4 changed files with 706 additions and 35 deletions
@@ -0,0 +1,304 @@
+"""Integration tests for the SEC fundamentals importer, driven through the real
+import framework with a fake SEC client (no network)."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from datetime import date, datetime, timezone
+
+import pytest
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from app.database import Base
+import app.models  # noqa: F401
+from app.models.fundamental_snapshot import FundamentalSnapshot
+from app.models.ticker import Ticker
+from app.services.data_import import STATUS_FAILED, STATUS_PROMOTED, run_import
+from app.services.sec_fundamentals_importer import SecFundamentalsImporter
+
+
+@pytest.fixture
+async def engine():
+    fd, path = tempfile.mkstemp(suffix=".db")
+    os.close(fd)
+    eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
+    async with eng.begin() as conn:
+        await conn.run_sync(Base.metadata.create_all)
+    try:
+        yield eng
+    finally:
+        await eng.dispose()
+        try:
+            os.unlink(path)
+        except OSError:
+            pass
+
+
+def _factory(engine):
+    return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
+
+
+# --- fixture SEC data (AAPL, cik 320193) -----------------------------------
+
+def _rev(start, end, val, fy, fp, accn):
+    return {"start": start, "end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
+
+
+def _shares(end, val, accn, fy, fp):
+    return {"end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
+
+
+def _companyfacts(rev_facts, share_facts):
+    return {
+        "cik": 320193,
+        "facts": {
+            "us-gaap": {"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": rev_facts}}},
+            "dei": {"EntityCommonStockSharesOutstanding": {"units": {"shares": share_facts}}},
+        },
+    }
+
+
+def _filing(accn, form, report, filed, accepted, is_xbrl=True):
+    return {"accession": accn, "form": form, "report_date": report, "filing_date": filed,
+            "acceptance_datetime": accepted, "is_xbrl": is_xbrl}
+
+
+CF_K = _rev("2024-09-29", "2025-09-27", 416161, 2025, "FY", "K")
+CF_Q1 = _rev("2025-09-28", "2025-12-27", 143756, 2026, "Q1", "Q")
+SH_K = _shares("2025-10-17", 14776, "K", 2025, "FY")
+SH_Q1 = _shares("2026-01-16", 14681, "Q", 2026, "Q1")
+
+SUB_FILINGS = [
+    _filing("K", "10-K", "2025-09-27", "2025-10-31", "2025-10-31T10:01:26.000Z"),
+    _filing("Q", "10-Q", "2025-12-27", "2026-01-30", "2026-01-30T11:01:00.000Z"),
+]
+
+
+def _submissions(filings):
+    return {"cik": 320193, "sic": "3571", "sic_description": "Electronic Computers",
+            "fiscal_year_end": "0926", "tickers": ["AAPL"], "filings": filings}
+
+
+class FakeSecClient:
+    def __init__(self, *, tickers, companyfacts, submissions, latest_index, daily=None):
+        self._tickers = tickers
+        self._cf = companyfacts
+        self._sub = submissions
+        self._latest = latest_index
+        self._daily = daily or {}
+
+    async def __aenter__(self):
+        return self
+
+    async def __aexit__(self, *a):
+        return False
+
+    async def company_tickers(self):
+        return dict(self._tickers)
+
+    async def latest_index_date(self, today=None):
+        return self._latest
+
+    async def daily_index(self, day):
+        return list(self._daily.get(day, []))
+
+    async def companyfacts(self, cik):
+        return self._cf[int(cik)]
+
+    async def submissions(self, cik, *, include_history=False):
+        return self._sub[int(cik)]
+
+
+def _importer(client, today=date(2026, 2, 1)):
+    return SecFundamentalsImporter(client_factory=lambda: client, today=today)
+
+
+async def _seed(factory, symbols):
+    async with factory() as s:
+        for sym in symbols:
+            s.add(Ticker(symbol=sym))
+        await s.commit()
+
+
+async def _count(factory, model):
+    async with factory() as s:
+        return (await s.execute(select(func.count()).select_from(model))).scalar_one()
+
+
+# ---------------------------------------------------------------------------
+
+
+async def test_backfill_inserts_snapshots_and_ticker_meta(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    client = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},
+        latest_index=date(2026, 1, 31),
+    )
+    run = await run_import(_importer(client), engine=engine)
+
+    assert run.status == STATUS_PROMOTED
+    assert run.source_max_date == date(2026, 1, 31)
+    assert await _count(factory, FundamentalSnapshot) == 2
+
+    async with factory() as s:
+        t = (await s.execute(select(Ticker))).scalar_one()
+        assert t.cik == "0000320193" and t.sic == "3571"
+        snaps = (await s.execute(select(FundamentalSnapshot))).scalars().all()
+    assert {x.fiscal_period for x in snaps} == {"FY", "Q1"}
+    assert all(x.import_run_id == run.id for x in snaps)
+    fy = next(x for x in snaps if x.fiscal_period == "FY")
+    assert fy.revenue == 416161 and fy.shares_outstanding == 14776
+
+
+async def test_incremental_adds_only_new_filing(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    backfill = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},
+        latest_index=date(2026, 1, 31),
+    )
+    await run_import(_importer(backfill), engine=engine)
+    assert await _count(factory, FundamentalSnapshot) == 2
+
+    # A new Q2 10-Q appears in the daily index and Company Facts.
+    cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A")
+    sh_q2 = _shares("2026-04-17", 14687, "Q2A", 2026, "Q2")
+    incr = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])},
+        submissions={320193: _submissions(SUB_FILINGS + [
+            _filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
+        latest_index=date(2026, 5, 2),
+        daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "Q2A"}]},
+    )
+    run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
+
+    assert run.status == STATUS_PROMOTED
+    assert await _count(factory, FundamentalSnapshot) == 3  # only Q2A added
+    async with factory() as s:
+        q2 = (await s.execute(
+            select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A")
+        )).scalar_one()
+    assert q2.fiscal_period == "Q2" and q2.revenue == 254940
+
+
+async def test_consistency_gate_fails_when_facts_lag_index(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    backfill = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},
+        latest_index=date(2026, 1, 31),
+    )
+    await run_import(_importer(backfill), engine=engine)
+
+    # Index + submissions list an XBRL filing "GHOST" that Company Facts lacks.
+    incr = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},  # no GHOST
+        submissions={320193: _submissions(SUB_FILINGS + [
+            _filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
+        latest_index=date(2026, 5, 2),
+        daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]},
+    )
+    run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
+
+    assert run.status == STATUS_FAILED
+    assert "Company Facts" in (run.error_details or "")
+    assert await _count(factory, FundamentalSnapshot) == 2  # nothing new written
+
+
+async def test_non_xbrl_amendment_skipped_not_failed(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    backfill = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},
+        latest_index=date(2026, 1, 31),
+    )
+    await run_import(_importer(backfill), engine=engine)
+
+    incr = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS + [
+            _filing("AMD", "10-K/A", "2025-09-27", "2026-05-01", "2026-05-01T10:01:00.000Z", is_xbrl=False)])},
+        latest_index=date(2026, 5, 2),
+        daily={date(2026, 5, 1): [{"form": "10-K/A", "cik": 320193, "accession": "AMD"}]},
+    )
+    run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
+
+    assert run.status == STATUS_PROMOTED  # non-XBRL amendment is skipped, not a failure
+    assert "skipped_non_xbrl" in (run.validation_json or "")
+    assert await _count(factory, FundamentalSnapshot) == 2
+
+
+async def test_failed_backfill_leaves_tickers_unwritten(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL", "MSFT", "NVDA"])  # 3 resolve, only AAPL yields rows
+    client = FakeSecClient(
+        tickers={"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810},
+        companyfacts={
+            320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
+            789019: {"cik": 789019, "facts": {}},   # no facts -> no rows
+            1045810: {"cik": 1045810, "facts": {}},
+        },
+        submissions={
+            320193: _submissions(SUB_FILINGS),
+            789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
+            1045810: {"cik": 1045810, "sic": "3674", "sic_description": "y", "filings": []},
+        },
+        latest_index=date(2026, 1, 31),
+    )
+    run = await run_import(_importer(client), engine=engine)
+
+    assert run.status == STATUS_FAILED  # coverage 1/3 < 50%
+    assert "coverage" in (run.error_details or "")
+    assert await _count(factory, FundamentalSnapshot) == 0
+    # read-only resolution: no ticker cik/sic written on a failed run
+    async with factory() as s:
+        assert all(t.cik is None and t.sic is None for t in (await s.execute(select(Ticker))).scalars())
+
+
+async def test_promote_conflict_reports_discrepancy_without_mutation(engine):
+    from app.services.sec_facts_parser import SnapshotRow
+    from app.services.sec_fundamentals_importer import StagedFundamentals
+    from app.services.sec_universe import ResolvedUniverse
+
+    factory = _factory(engine)
+    utc = timezone.utc
+    async with factory() as s:  # pre-existing immutable snapshot K (revenue 100, run 1)
+        s.add(FundamentalSnapshot(
+            cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31),
+            accepted_at=datetime(2025, 10, 31, tzinfo=utc), period_end=date(2025, 9, 27),
+            fiscal_year=2025, fiscal_period="FY", revenue=100.0, import_run_id=1,
+            created_at=datetime(2025, 10, 31, tzinfo=utc)))
+        await s.commit()
+
+    k_diff = SnapshotRow(cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31),
+                         accepted_at=datetime(2025, 10, 31, tzinfo=utc), period_end=date(2025, 9, 27),
+                         fiscal_year=2025, fiscal_period="FY", revenue=999.0)  # differs
+    n_new = SnapshotRow(cik="0000320193", accession="N", form="10-Q", filed_date=date(2026, 1, 30),
+                        accepted_at=datetime(2026, 1, 30, tzinfo=utc), period_end=date(2025, 12, 27),
+                        fiscal_year=2026, fiscal_period="Q1", revenue=143.0)
+    staged = StagedFundamentals(resolved=ResolvedUniverse(), rows=[k_diff, n_new])
+
+    imp = SecFundamentalsImporter(client_factory=lambda: None)
+    async with factory() as s:
+        counts = await imp.promote(s, staged, run_id=2)
+        await s.commit()
+
+    assert counts["inserted"] == 1 and counts["discrepancies"] == 1
+    async with factory() as s:
+        k = (await s.execute(select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "K"))).scalar_one()
+        assert k.revenue == 100.0 and k.import_run_id == 1  # immutable — not overwritten
+        assert (await s.execute(select(func.count()).select_from(FundamentalSnapshot))).scalar_one() == 2