dennisthiessenandClaude Opus 4.8
8dcdcac2a6
1. Removed the 45-day index-walk cap: it discarded the older part of a long outage while still advancing source_max_date, permanently losing filings. The walk now covers every unprocessed date (a large gap is one-time cost). 2. Discrepancy detection meets the immutability contract: it compares ALL source snapshot fields (not five), read-only during stage/validate, reports the differing accessions + fields in validation_json, and promote emits a warning system event (in-transaction) — never mutating the stored row. 3. Malformed companyfacts (missing facts/units structure) are recorded separately and FAIL validation, instead of silently degrading to skipped rows that the 50% backfill coverage floor could still pass. Also corrected the stale "sum share classes" / DEI-only wording in the snapshot model docstring and the A3 design doc to describe the us-gaap fallback. Tests: +4 regressions (>45-day gap loses nothing, newly-added issuer backfills without filing, malformed payload fails, shares discrepancy detected + evented). 23 passed, 1 skipped. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
388 lines
16 KiB
Python
388 lines
16 KiB
Python
"""Integration tests for the SEC fundamentals importer, driven through the real
|
|
import framework with a fake SEC client (no network)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
from datetime import date, datetime, timezone
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.database import Base
|
|
import app.models # noqa: F401
|
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
|
from app.models.ticker import Ticker
|
|
from app.services.data_import import STATUS_FAILED, STATUS_PROMOTED, run_import
|
|
from app.services.sec_fundamentals_importer import SecFundamentalsImporter
|
|
|
|
|
|
@pytest.fixture
|
|
async def engine():
|
|
fd, path = tempfile.mkstemp(suffix=".db")
|
|
os.close(fd)
|
|
eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
|
|
async with eng.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
try:
|
|
yield eng
|
|
finally:
|
|
await eng.dispose()
|
|
try:
|
|
os.unlink(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _factory(engine):
|
|
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
|
|
# --- fixture SEC data (AAPL, cik 320193) -----------------------------------
|
|
|
|
def _rev(start, end, val, fy, fp, accn):
|
|
return {"start": start, "end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
|
|
|
|
|
|
def _shares(end, val, accn, fy, fp):
|
|
return {"end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
|
|
|
|
|
|
def _companyfacts(rev_facts, share_facts, cik=320193):
|
|
return {
|
|
"cik": cik,
|
|
"facts": {
|
|
"us-gaap": {"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": rev_facts}}},
|
|
"dei": {"EntityCommonStockSharesOutstanding": {"units": {"shares": share_facts}}},
|
|
},
|
|
}
|
|
|
|
|
|
def _filing(accn, form, report, filed, accepted, is_xbrl=True):
|
|
return {"accession": accn, "form": form, "report_date": report, "filing_date": filed,
|
|
"acceptance_datetime": accepted, "is_xbrl": is_xbrl}
|
|
|
|
|
|
CF_K = _rev("2024-09-29", "2025-09-27", 416161, 2025, "FY", "K")
|
|
CF_Q1 = _rev("2025-09-28", "2025-12-27", 143756, 2026, "Q1", "Q")
|
|
SH_K = _shares("2025-10-17", 14776, "K", 2025, "FY")
|
|
SH_Q1 = _shares("2026-01-16", 14681, "Q", 2026, "Q1")
|
|
|
|
SUB_FILINGS = [
|
|
_filing("K", "10-K", "2025-09-27", "2025-10-31", "2025-10-31T10:01:26.000Z"),
|
|
_filing("Q", "10-Q", "2025-12-27", "2026-01-30", "2026-01-30T11:01:00.000Z"),
|
|
]
|
|
|
|
|
|
def _submissions(filings):
|
|
return {"cik": 320193, "sic": "3571", "sic_description": "Electronic Computers",
|
|
"fiscal_year_end": "0926", "tickers": ["AAPL"], "filings": filings}
|
|
|
|
|
|
class FakeSecClient:
|
|
def __init__(self, *, tickers, companyfacts, submissions, latest_index, daily=None):
|
|
self._tickers = tickers
|
|
self._cf = companyfacts
|
|
self._sub = submissions
|
|
self._latest = latest_index
|
|
self._daily = daily or {}
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *a):
|
|
return False
|
|
|
|
async def company_tickers(self):
|
|
return dict(self._tickers)
|
|
|
|
async def latest_index_date(self, today=None):
|
|
return self._latest
|
|
|
|
async def daily_index(self, day):
|
|
return list(self._daily.get(day, []))
|
|
|
|
async def companyfacts(self, cik):
|
|
return self._cf[int(cik)]
|
|
|
|
async def submissions(self, cik, *, include_history=False):
|
|
return self._sub[int(cik)]
|
|
|
|
|
|
def _importer(client, today=date(2026, 2, 1)):
|
|
return SecFundamentalsImporter(client_factory=lambda: client, today=today)
|
|
|
|
|
|
async def _seed(factory, symbols):
|
|
async with factory() as s:
|
|
for sym in symbols:
|
|
s.add(Ticker(symbol=sym))
|
|
await s.commit()
|
|
|
|
|
|
async def _count(factory, model):
|
|
async with factory() as s:
|
|
return (await s.execute(select(func.count()).select_from(model))).scalar_one()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_backfill_inserts_snapshots_and_ticker_meta(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED
|
|
assert run.source_max_date == date(2026, 1, 31)
|
|
assert await _count(factory, FundamentalSnapshot) == 2
|
|
|
|
async with factory() as s:
|
|
t = (await s.execute(select(Ticker))).scalar_one()
|
|
assert t.cik == "0000320193" and t.sic == "3571"
|
|
snaps = (await s.execute(select(FundamentalSnapshot))).scalars().all()
|
|
assert {x.fiscal_period for x in snaps} == {"FY", "Q1"}
|
|
assert all(x.import_run_id == run.id for x in snaps)
|
|
fy = next(x for x in snaps if x.fiscal_period == "FY")
|
|
assert fy.revenue == 416161 and fy.shares_outstanding == 14776
|
|
|
|
|
|
async def test_incremental_adds_only_new_filing(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
assert await _count(factory, FundamentalSnapshot) == 2
|
|
|
|
# A new Q2 10-Q appears in the daily index and Company Facts.
|
|
cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A")
|
|
sh_q2 = _shares("2026-04-17", 14687, "Q2A", 2026, "Q2")
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])},
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "Q2A"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED
|
|
assert await _count(factory, FundamentalSnapshot) == 3 # only Q2A added
|
|
async with factory() as s:
|
|
q2 = (await s.execute(
|
|
select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A")
|
|
)).scalar_one()
|
|
assert q2.fiscal_period == "Q2" and q2.revenue == 254940
|
|
|
|
|
|
async def test_consistency_gate_fails_when_facts_lag_index(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
# Index + submissions list an XBRL filing "GHOST" that Company Facts lacks.
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # no GHOST
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_FAILED
|
|
assert "Company Facts" in (run.error_details or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 2 # nothing new written
|
|
|
|
|
|
async def test_non_xbrl_amendment_skipped_not_failed(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("AMD", "10-K/A", "2025-09-27", "2026-05-01", "2026-05-01T10:01:00.000Z", is_xbrl=False)])},
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [{"form": "10-K/A", "cik": 320193, "accession": "AMD"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED # non-XBRL amendment is skipped, not a failure
|
|
assert "skipped_non_xbrl" in (run.validation_json or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 2
|
|
|
|
|
|
async def test_failed_backfill_leaves_tickers_unwritten(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL", "MSFT", "NVDA"]) # 3 resolve, only AAPL yields rows
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810},
|
|
companyfacts={
|
|
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
|
789019: {"cik": 789019, "facts": {}}, # no facts -> no rows
|
|
1045810: {"cik": 1045810, "facts": {}},
|
|
},
|
|
submissions={
|
|
320193: _submissions(SUB_FILINGS),
|
|
789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
|
|
1045810: {"cik": 1045810, "sic": "3674", "sic_description": "y", "filings": []},
|
|
},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
|
|
assert run.status == STATUS_FAILED # coverage 1/3 < 50%
|
|
assert "coverage" in (run.error_details or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 0
|
|
# read-only resolution: no ticker cik/sic written on a failed run
|
|
async with factory() as s:
|
|
assert all(t.cik is None and t.sic is None for t in (await s.execute(select(Ticker))).scalars())
|
|
|
|
|
|
async def test_index_gap_over_45_days_loses_no_filings(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
# 74-day gap; the filing sits in the OLD part (>45d before latest).
|
|
cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "OLD")
|
|
sh_q2 = _shares("2026-04-17", 14687, "OLD", 2026, "Q2")
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])},
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("OLD", "10-Q", "2026-03-28", "2026-02-10", "2026-02-10T10:01:00.000Z")])},
|
|
latest_index=date(2026, 4, 15),
|
|
daily={date(2026, 2, 10): [{"form": "10-Q", "cik": 320193, "accession": "OLD"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 4, 16)), engine=engine)
|
|
assert run.status == STATUS_PROMOTED
|
|
assert await _count(factory, FundamentalSnapshot) == 3 # the old-gap filing was NOT lost
|
|
|
|
|
|
async def test_newly_added_issuer_backfills_without_filing(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
# MSFT added to the universe later; it did NOT file (not in the daily index).
|
|
await _seed(factory, ["MSFT"])
|
|
msft_rev = _rev("2024-07-01", "2025-06-30", 270000, 2025, "FY", "M")
|
|
msft_sh = _shares("2025-07-15", 7400, "M", 2025, "FY")
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193, "MSFT": 789019},
|
|
companyfacts={
|
|
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
|
789019: _companyfacts([msft_rev], [msft_sh], cik=789019),
|
|
},
|
|
submissions={
|
|
320193: _submissions(SUB_FILINGS),
|
|
789019: {"cik": 789019, "sic": "7372", "sic_description": "Prepackaged Software",
|
|
"filings": [_filing("M", "10-K", "2025-06-30", "2025-07-30", "2025-07-30T10:00:00.000Z")]},
|
|
},
|
|
latest_index=date(2026, 2, 3),
|
|
daily={}, # MSFT did not file
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 2, 4)), engine=engine)
|
|
assert run.status == STATUS_PROMOTED
|
|
async with factory() as s:
|
|
msft = (await s.execute(
|
|
select(FundamentalSnapshot).where(FundamentalSnapshot.cik == "0000789019")
|
|
)).scalars().all()
|
|
assert len(msft) == 1 and msft[0].revenue == 270000 # full-history backfill despite no filing
|
|
|
|
|
|
async def test_malformed_companyfacts_fails_validation(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL", "MSFT"])
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193, "MSFT": 789019},
|
|
companyfacts={
|
|
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
|
789019: {"cik": 789019}, # malformed — no "facts" structure
|
|
},
|
|
submissions={
|
|
320193: _submissions(SUB_FILINGS),
|
|
789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
|
|
},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
assert run.status == STATUS_FAILED
|
|
assert "malformed" in (run.error_details or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 0 # nothing promoted
|
|
|
|
|
|
async def test_discrepancy_in_shares_is_detected_and_reported(engine):
|
|
from app.models.system_event import SystemEvent
|
|
utc = timezone.utc
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
# Pre-store accession K matching what the parser will produce EXCEPT shares.
|
|
async with factory() as s:
|
|
s.add(FundamentalSnapshot(
|
|
cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31),
|
|
accepted_at=datetime(2025, 10, 31, 10, 1, 26, tzinfo=utc), period_start=date(2024, 9, 29),
|
|
period_end=date(2025, 9, 27), fiscal_year=2025, fiscal_period="FY", revenue=416161.0,
|
|
shares_outstanding=999.0, import_run_id=1, created_at=datetime(2025, 10, 31, tzinfo=utc)))
|
|
await s.commit()
|
|
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # SH_K = 14776 != 999
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED # a discrepancy is reported, not a failure
|
|
assert '"discrepancy_count": 1' in (run.validation_json or "")
|
|
assert "shares_outstanding" in (run.validation_json or "")
|
|
async with factory() as s:
|
|
k = (await s.execute(select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "K"))).scalar_one()
|
|
assert k.shares_outstanding == 999.0 and k.import_run_id == 1 # immutable — not overwritten
|
|
events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all()
|
|
assert len(events) == 1 and events[0].severity == "warning"
|