857 lines
34 KiB
Python
857 lines
34 KiB
Python
"""Integration tests for the SEC fundamentals importer, driven through the real
|
|
import framework with a fake SEC client (no network)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import tempfile
|
|
from datetime import date, datetime, timezone
|
|
|
|
import pytest
|
|
from sqlalchemy import func, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
|
|
|
from app.database import Base
|
|
import app.models # noqa: F401
|
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
|
from app.models.sec_filing_gap import SecFilingGap
|
|
from app.models.system_event import SystemEvent
|
|
from app.models.ticker import Ticker
|
|
from app.services.data_import import (
|
|
STATUS_DEFERRED,
|
|
STATUS_FAILED,
|
|
STATUS_PROMOTED,
|
|
run_import,
|
|
)
|
|
from app.services.sec_fundamentals_importer import (
|
|
SecFundamentalsImporter,
|
|
StagedFundamentals,
|
|
)
|
|
from app.services.sec_universe import ResolvedUniverse
|
|
|
|
|
|
@pytest.fixture
|
|
async def engine():
|
|
fd, path = tempfile.mkstemp(suffix=".db")
|
|
os.close(fd)
|
|
eng = create_async_engine(f"sqlite+aiosqlite:///{path}")
|
|
async with eng.begin() as conn:
|
|
await conn.run_sync(Base.metadata.create_all)
|
|
try:
|
|
yield eng
|
|
finally:
|
|
await eng.dispose()
|
|
try:
|
|
os.unlink(path)
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def _factory(engine):
|
|
return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
|
|
|
|
|
|
# --- fixture SEC data (AAPL, cik 320193) -----------------------------------
|
|
|
|
def _rev(start, end, val, fy, fp, accn):
|
|
return {"start": start, "end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
|
|
|
|
|
|
def _shares(end, val, accn, fy, fp):
|
|
return {"end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"}
|
|
|
|
|
|
def _companyfacts(rev_facts, share_facts, cik=320193):
|
|
return {
|
|
"cik": cik,
|
|
"facts": {
|
|
"us-gaap": {"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": rev_facts}}},
|
|
"dei": {"EntityCommonStockSharesOutstanding": {"units": {"shares": share_facts}}},
|
|
},
|
|
}
|
|
|
|
|
|
def _filing(accn, form, report, filed, accepted, is_xbrl=True):
|
|
return {"accession": accn, "form": form, "report_date": report, "filing_date": filed,
|
|
"acceptance_datetime": accepted, "is_xbrl": is_xbrl}
|
|
|
|
|
|
CF_K = _rev("2024-09-29", "2025-09-27", 416161, 2025, "FY", "K")
|
|
CF_Q1 = _rev("2025-09-28", "2025-12-27", 143756, 2026, "Q1", "Q")
|
|
SH_K = _shares("2025-10-17", 14776, "K", 2025, "FY")
|
|
SH_Q1 = _shares("2026-01-16", 14681, "Q", 2026, "Q1")
|
|
|
|
SUB_FILINGS = [
|
|
_filing("K", "10-K", "2025-09-27", "2025-10-31", "2025-10-31T10:01:26.000Z"),
|
|
_filing("Q", "10-Q", "2025-12-27", "2026-01-30", "2026-01-30T11:01:00.000Z"),
|
|
]
|
|
|
|
|
|
def _submissions(filings):
|
|
return {"cik": 320193, "sic": "3571", "sic_description": "Electronic Computers",
|
|
"fiscal_year_end": "0926", "tickers": ["AAPL"], "filings": filings}
|
|
|
|
|
|
class FakeSecClient:
|
|
def __init__(self, *, tickers, companyfacts, submissions, latest_index, daily=None):
|
|
self._tickers = tickers
|
|
self._cf = companyfacts
|
|
self._sub = submissions
|
|
self._latest = latest_index
|
|
self._daily = daily or {}
|
|
|
|
async def __aenter__(self):
|
|
return self
|
|
|
|
async def __aexit__(self, *a):
|
|
return False
|
|
|
|
async def company_tickers(self):
|
|
return dict(self._tickers)
|
|
|
|
async def latest_index_date(self, today=None):
|
|
return self._latest
|
|
|
|
async def daily_index(self, day):
|
|
return list(self._daily.get(day, []))
|
|
|
|
async def companyfacts(self, cik):
|
|
return self._cf[int(cik)]
|
|
|
|
async def submissions(self, cik, *, include_history=False):
|
|
return self._sub[int(cik)]
|
|
|
|
|
|
def _importer(client, today=date(2026, 2, 1)):
|
|
return SecFundamentalsImporter(client_factory=lambda: client, today=today)
|
|
|
|
|
|
async def _seed(factory, symbols):
|
|
async with factory() as s:
|
|
for sym in symbols:
|
|
s.add(Ticker(symbol=sym))
|
|
await s.commit()
|
|
|
|
|
|
async def _count(factory, model):
|
|
async with factory() as s:
|
|
return (await s.execute(select(func.count()).select_from(model))).scalar_one()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_backfill_inserts_snapshots_and_ticker_meta(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED
|
|
assert run.source_max_date == date(2026, 1, 31)
|
|
assert await _count(factory, FundamentalSnapshot) == 2
|
|
|
|
async with factory() as s:
|
|
t = (await s.execute(select(Ticker))).scalar_one()
|
|
assert t.cik == "0000320193" and t.sic == "3571"
|
|
snaps = (await s.execute(select(FundamentalSnapshot))).scalars().all()
|
|
assert {x.fiscal_period for x in snaps} == {"FY", "Q1"}
|
|
assert all(x.import_run_id == run.id for x in snaps)
|
|
fy = next(x for x in snaps if x.fiscal_period == "FY")
|
|
assert fy.revenue == 416161 and fy.shares_outstanding == 14776
|
|
|
|
|
|
async def test_incremental_adds_only_new_filing(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
assert await _count(factory, FundamentalSnapshot) == 2
|
|
|
|
# A new Q2 10-Q appears in the daily index and Company Facts.
|
|
cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A")
|
|
sh_q2 = _shares("2026-04-17", 14687, "Q2A", 2026, "Q2")
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])},
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "Q2A"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED
|
|
assert await _count(factory, FundamentalSnapshot) == 3 # only Q2A added
|
|
async with factory() as s:
|
|
q2 = (await s.execute(
|
|
select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A")
|
|
)).scalar_one()
|
|
assert q2.fiscal_period == "Q2" and q2.revenue == 254940
|
|
|
|
|
|
async def test_consistency_gate_defers_without_alert_when_facts_lag_index(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
# Index + submissions list an XBRL filing "GHOST" that Company Facts lacks.
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # no GHOST
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_DEFERRED
|
|
# The gate blocks every later run until it clears, so run history still has
|
|
# to name the filing and say why it could not be resolved.
|
|
details = run.error_details or ""
|
|
assert "GHOST" in details and "not_in_companyfacts" in details
|
|
assert "2026-05-01" in details # index date the filing was seen on
|
|
summary = json.loads(run.validation_json or "{}")
|
|
assert summary["missing_xbrl_count"] == 1
|
|
assert summary["missing_xbrl"][0]["accession"] == "GHOST"
|
|
assert summary["missing_xbrl"][0]["form"] == "10-Q"
|
|
assert await _count(factory, FundamentalSnapshot) == 2 # nothing new written
|
|
assert await _count(factory, SystemEvent) == 0 # expected SEC lag does not alert
|
|
|
|
|
|
async def test_companyfacts_lag_does_not_mask_second_validation_failure():
|
|
importer = SecFundamentalsImporter(today=date(2026, 5, 3))
|
|
importer._latest_index_date = date(2026, 5, 2)
|
|
staged = StagedFundamentals(
|
|
resolved=ResolvedUniverse(),
|
|
missing_xbrl=[{
|
|
"cik": "0000320193",
|
|
"accession": "GHOST",
|
|
"form": "10-Q",
|
|
"index_date": date(2026, 5, 1),
|
|
"age_days": 2,
|
|
"reason": "not_in_companyfacts",
|
|
}],
|
|
invalid_payloads=[{
|
|
"cik": "0000789019",
|
|
"reason": "missing facts structure",
|
|
}],
|
|
)
|
|
|
|
result = await importer.validate(None, staged)
|
|
|
|
assert not result.ok
|
|
assert not result.retryable
|
|
assert len(result.messages) == 2
|
|
|
|
|
|
async def test_deferred_alert_names_aged_out_accessions_separately():
|
|
importer = SecFundamentalsImporter(today=date(2026, 5, 6))
|
|
importer._latest_index_date = date(2026, 5, 5)
|
|
staged = StagedFundamentals(
|
|
resolved=ResolvedUniverse(),
|
|
missing_xbrl=[
|
|
{
|
|
"cik": "0000320193",
|
|
"accession": "YOUNG",
|
|
"form": "10-Q",
|
|
"index_date": date(2026, 5, 5),
|
|
"age_days": 1,
|
|
"reason": "not_in_companyfacts",
|
|
},
|
|
{
|
|
"cik": "0000789019",
|
|
"accession": "AGED-OUT",
|
|
"form": "10-Q",
|
|
"index_date": date(2026, 5, 1),
|
|
"age_days": 5,
|
|
"reason": "not_in_companyfacts",
|
|
},
|
|
],
|
|
)
|
|
|
|
result = await importer.validate(None, staged)
|
|
|
|
assert result.retryable
|
|
assert len(result.messages) == 1 and "YOUNG" in result.messages[0]
|
|
assert len(result.deferred_alert_messages) == 1
|
|
assert "AGED-OUT" in result.deferred_alert_messages[0]
|
|
|
|
|
|
async def test_gate_separates_missing_submissions_from_missing_facts(engine):
|
|
"""An index row the issuer's own filing list does not carry is a different
|
|
failure from a Company-Facts lag, and must not be reported as one."""
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)}, # submissions never lists ORPHAN
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "ORPHAN"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_FAILED
|
|
assert "ORPHAN" in (run.error_details or "")
|
|
assert "not_in_submissions" in (run.error_details or "")
|
|
summary = json.loads(run.validation_json or "{}")
|
|
assert summary["missing_xbrl"][0]["reason"] == "not_in_submissions"
|
|
|
|
|
|
def _coregistrant_client(share_fact):
|
|
"""Incremental client where Q2A's facts landed in co-registrant 99999's file
|
|
instead of the filer's own — the NEE-via-FPL / DOW-via-Dow-Chemical shape."""
|
|
cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A")
|
|
return FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={
|
|
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), # filer's own: no Q2A
|
|
99999: _companyfacts([cf_q2], [share_fact], cik=99999),
|
|
},
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [
|
|
# one combined filing, listed by the index under both co-registrants
|
|
{"form": "10-Q", "cik": 320193, "accession": "Q2A"},
|
|
{"form": "10-Q", "cik": 99999, "accession": "Q2A"},
|
|
]},
|
|
)
|
|
|
|
|
|
async def test_recovers_facts_misfiled_under_coregistrant(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
# 14687 shares is continuous with the issuer's own history (14681 last quarter).
|
|
incr = _coregistrant_client(_shares("2026-04-17", 14687, "Q2A", 2026, "Q2"))
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED
|
|
summary = json.loads(run.validation_json or "{}")
|
|
assert summary["recovered_count"] == 1
|
|
assert summary["recovered_from_coregistrant"][0]["source_cik"] == "0000099999"
|
|
assert summary["missing_xbrl_count"] == 0
|
|
|
|
async with factory() as s:
|
|
q2 = (await s.execute(
|
|
select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A")
|
|
)).scalar_one()
|
|
codes = (await s.execute(select(SystemEvent.code))).scalars().all()
|
|
# Stamped to the issuer that filed, NOT the co-registrant whose file it came from.
|
|
assert q2.cik == "0000320193"
|
|
assert q2.revenue == 254940 and q2.shares_outstanding == 14687
|
|
assert "coregistrant_recovery" not in codes
|
|
|
|
|
|
async def test_coregistrant_recovery_rejects_discontinuous_share_count(engine):
|
|
"""A co-registrant shell's standalone facts must never be stored as the
|
|
parent's — a token float is the signature and it has to be refused."""
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
incr = _coregistrant_client(_shares("2026-04-17", 100, "Q2A", 2026, "Q2"))
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_FAILED
|
|
assert "coregistrant_facts_rejected" in (run.error_details or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 2 # nothing recovered
|
|
|
|
|
|
async def test_unresolved_filing_stops_blocking_after_retry_window(engine):
|
|
"""A filing SEC has misfiled must not wedge every later import forever."""
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # no GHOST
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]},
|
|
)
|
|
# 9 days after the index date — well past the retry window.
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 10)), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED # promoted around it, not blocked by it
|
|
assert run.source_max_date == date(2026, 5, 2) # and the index advances
|
|
summary = json.loads(run.validation_json or "{}")
|
|
assert summary["missing_xbrl_count"] == 1 and summary["missing_xbrl_blocking"] == 0
|
|
assert await _count(factory, SecFilingGap) == 1
|
|
|
|
async with factory() as s:
|
|
events = (await s.execute(select(SystemEvent))).scalars().all()
|
|
unresolved = [e for e in events if e.code == "unresolved_filing"]
|
|
assert len(unresolved) == 1 and "GHOST" in unresolved[0].message
|
|
assert "automatic SEC retry" in unresolved[0].message
|
|
|
|
# The next scheduled run retries even though the SEC daily-index revision
|
|
# has not changed. Company Facts is a separate SEC product and may catch up
|
|
# independently, so the generic revision no-op must not suppress this work.
|
|
still_missing_run = await run_import(
|
|
_importer(incr, today=date(2026, 5, 11)),
|
|
engine=engine,
|
|
)
|
|
assert still_missing_run.status == STATUS_PROMOTED
|
|
assert still_missing_run.revision is None
|
|
assert await _count(factory, SecFilingGap) == 1
|
|
|
|
# A later normal scheduled import retries only the queued issuer. Once SEC
|
|
# publishes the accession in Company Facts, it is inserted and unblocked
|
|
# without a full-universe reparse or operator action.
|
|
cf_ghost = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "GHOST")
|
|
sh_ghost = _shares("2026-04-17", 14687, "GHOST", 2026, "Q2")
|
|
healed = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={
|
|
320193: _companyfacts(
|
|
[CF_K, CF_Q1, cf_ghost],
|
|
[SH_K, SH_Q1, sh_ghost],
|
|
)
|
|
},
|
|
submissions={
|
|
320193: _submissions(SUB_FILINGS + [
|
|
_filing(
|
|
"GHOST",
|
|
"10-Q",
|
|
"2026-03-28",
|
|
"2026-05-01",
|
|
"2026-05-01T10:01:00.000Z",
|
|
)
|
|
])
|
|
},
|
|
latest_index=date(2026, 5, 2),
|
|
)
|
|
healed_run = await run_import(
|
|
_importer(healed, today=date(2026, 5, 12)),
|
|
engine=engine,
|
|
)
|
|
|
|
assert healed_run.status == STATUS_PROMOTED
|
|
assert await _count(factory, FundamentalSnapshot) == 3
|
|
assert await _count(factory, SecFilingGap) == 0
|
|
|
|
|
|
async def test_non_xbrl_amendment_skipped_not_failed(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("AMD", "10-K/A", "2025-09-27", "2026-05-01", "2026-05-01T10:01:00.000Z", is_xbrl=False)])},
|
|
latest_index=date(2026, 5, 2),
|
|
daily={date(2026, 5, 1): [{"form": "10-K/A", "cik": 320193, "accession": "AMD"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED # non-XBRL amendment is skipped, not a failure
|
|
assert "skipped_non_xbrl" in (run.validation_json or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 2
|
|
|
|
|
|
async def test_failed_backfill_leaves_tickers_unwritten(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL", "MSFT", "NVDA"]) # 3 resolve, only AAPL yields rows
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810},
|
|
companyfacts={
|
|
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
|
789019: {"cik": 789019, "facts": {}}, # no facts -> no rows
|
|
1045810: {"cik": 1045810, "facts": {}},
|
|
},
|
|
submissions={
|
|
320193: _submissions(SUB_FILINGS),
|
|
789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
|
|
1045810: {"cik": 1045810, "sic": "3674", "sic_description": "y", "filings": []},
|
|
},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
|
|
assert run.status == STATUS_FAILED # coverage 1/3 < 50%
|
|
assert "coverage" in (run.error_details or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 0
|
|
# read-only resolution: no ticker cik/sic written on a failed run
|
|
async with factory() as s:
|
|
assert all(t.cik is None and t.sic is None for t in (await s.execute(select(Ticker))).scalars())
|
|
|
|
|
|
async def test_index_gap_over_45_days_loses_no_filings(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
# 74-day gap; the filing sits in the OLD part (>45d before latest).
|
|
cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "OLD")
|
|
sh_q2 = _shares("2026-04-17", 14687, "OLD", 2026, "Q2")
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])},
|
|
submissions={320193: _submissions(SUB_FILINGS + [
|
|
_filing("OLD", "10-Q", "2026-03-28", "2026-02-10", "2026-02-10T10:01:00.000Z")])},
|
|
latest_index=date(2026, 4, 15),
|
|
daily={date(2026, 2, 10): [{"form": "10-Q", "cik": 320193, "accession": "OLD"}]},
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 4, 16)), engine=engine)
|
|
assert run.status == STATUS_PROMOTED
|
|
assert await _count(factory, FundamentalSnapshot) == 3 # the old-gap filing was NOT lost
|
|
|
|
|
|
async def test_newly_added_issuer_backfills_without_filing(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
backfill = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
await run_import(_importer(backfill), engine=engine)
|
|
|
|
# MSFT added to the universe later; it did NOT file (not in the daily index).
|
|
await _seed(factory, ["MSFT"])
|
|
msft_rev = _rev("2024-07-01", "2025-06-30", 270000, 2025, "FY", "M")
|
|
msft_sh = _shares("2025-07-15", 7400, "M", 2025, "FY")
|
|
incr = FakeSecClient(
|
|
tickers={"AAPL": 320193, "MSFT": 789019},
|
|
companyfacts={
|
|
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
|
789019: _companyfacts([msft_rev], [msft_sh], cik=789019),
|
|
},
|
|
submissions={
|
|
320193: _submissions(SUB_FILINGS),
|
|
789019: {"cik": 789019, "sic": "7372", "sic_description": "Prepackaged Software",
|
|
"filings": [_filing("M", "10-K", "2025-06-30", "2025-07-30", "2025-07-30T10:00:00.000Z")]},
|
|
},
|
|
# SAME index date as the prior run and no filing: only the universe
|
|
# fingerprint (MSFT added) changes the revision, so this proves the
|
|
# fingerprint alone prevents starvation.
|
|
latest_index=date(2026, 1, 31),
|
|
daily={}, # MSFT did not file
|
|
)
|
|
run = await run_import(_importer(incr, today=date(2026, 2, 1)), engine=engine)
|
|
assert run.status == STATUS_PROMOTED
|
|
async with factory() as s:
|
|
msft = (await s.execute(
|
|
select(FundamentalSnapshot).where(FundamentalSnapshot.cik == "0000789019")
|
|
)).scalars().all()
|
|
assert len(msft) == 1 and msft[0].revenue == 270000 # full-history backfill despite no filing
|
|
|
|
|
|
async def test_malformed_companyfacts_fails_validation(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL", "MSFT"])
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193, "MSFT": 789019},
|
|
companyfacts={
|
|
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
|
789019: {"cik": 789019}, # malformed — no "facts" structure
|
|
},
|
|
submissions={
|
|
320193: _submissions(SUB_FILINGS),
|
|
789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
|
|
},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
assert run.status == STATUS_FAILED
|
|
assert "malformed" in (run.error_details or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 0 # nothing promoted
|
|
|
|
|
|
async def test_missing_units_structure_fails_validation(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL", "MSFT"])
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193, "MSFT": 789019},
|
|
companyfacts={
|
|
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
|
|
# facts present, but a concept is missing its units mapping
|
|
789019: {"cik": 789019, "facts": {"us-gaap": {"Revenues": {"label": "Revenues"}}}},
|
|
},
|
|
submissions={
|
|
320193: _submissions(SUB_FILINGS),
|
|
789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
|
|
},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
assert run.status == STATUS_FAILED
|
|
assert "malformed" in (run.error_details or "")
|
|
assert "units" in (run.validation_json or "")
|
|
assert await _count(factory, FundamentalSnapshot) == 0
|
|
|
|
|
|
async def test_discrepancy_in_shares_is_detected_and_reported(engine):
|
|
from app.models.system_event import SystemEvent
|
|
utc = timezone.utc
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
# Pre-store accession K matching what the parser will produce EXCEPT shares.
|
|
async with factory() as s:
|
|
s.add(FundamentalSnapshot(
|
|
cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31),
|
|
accepted_at=datetime(2025, 10, 31, 10, 1, 26, tzinfo=utc), period_start=date(2024, 9, 29),
|
|
period_end=date(2025, 9, 27), fiscal_year=2025, fiscal_period="FY", revenue=416161.0,
|
|
shares_outstanding=999.0, import_run_id=1, created_at=datetime(2025, 10, 31, tzinfo=utc)))
|
|
await s.commit()
|
|
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # SH_K = 14776 != 999
|
|
submissions={320193: _submissions(SUB_FILINGS)},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
run = await run_import(_importer(client), engine=engine)
|
|
|
|
assert run.status == STATUS_PROMOTED # a discrepancy is reported, not a failure
|
|
assert '"discrepancy_count": 1' in (run.validation_json or "")
|
|
assert "shares_outstanding" in (run.validation_json or "")
|
|
async with factory() as s:
|
|
k = (await s.execute(select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "K"))).scalar_one()
|
|
assert k.shares_outstanding == 999.0 and k.import_run_id == 1 # immutable — not overwritten
|
|
events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all()
|
|
assert len(events) == 1 and events[0].severity == "warning"
|
|
|
|
|
|
# --- reparse: rewriting rows a fixed parser reconstructs differently --------
|
|
|
|
# A 4-4-5 filer's YTD-Q3 span (36 weeks = 251 days). The old 20-day tolerance
|
|
# around 273 rejected it and stored revenue=None; 25 accepts it. Reparsing with
|
|
# the fixed parser is exactly the situation this mode exists for.
|
|
CF_Q3_445 = _rev("2025-09-01", "2026-05-10", 207431, 2026, "Q3", "Q3F")
|
|
SUB_445 = [_filing("Q3F", "10-Q", "2026-05-10", "2026-06-01", "2026-06-01T10:01:00.000Z")]
|
|
|
|
|
|
def _445_client():
|
|
return FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_Q3_445], [_shares("2026-05-15", 100, "Q3F", 2026, "Q3")])},
|
|
submissions={320193: _submissions(SUB_445)},
|
|
latest_index=date(2026, 6, 1),
|
|
)
|
|
|
|
|
|
async def _import_with_old_tolerance(engine, monkeypatch):
|
|
"""Seed the DB the way the pre-fix parser did: Q3 revenue rejected -> null."""
|
|
from app.services import sec_facts_parser
|
|
|
|
monkeypatch.setattr(sec_facts_parser, "_YTD_TOLERANCE_DAYS", 20)
|
|
run = await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
|
|
monkeypatch.undo()
|
|
return run
|
|
|
|
|
|
async def test_reparse_rewrites_rows_the_fixed_parser_reads_differently(engine, monkeypatch):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
first = await _import_with_old_tolerance(engine, monkeypatch)
|
|
|
|
async with factory() as s:
|
|
stale = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
|
assert stale.revenue is None, "precondition: the old parser stored a null"
|
|
|
|
# Reparse with the current (fixed) parser. force=True because SEC has not
|
|
# changed -- the staleness is on our side, so the revision gate would no-op.
|
|
run = await run_import(
|
|
SecFundamentalsImporter(
|
|
client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
|
|
),
|
|
engine=engine,
|
|
force=True,
|
|
)
|
|
|
|
assert run.status == STATUS_PROMOTED
|
|
assert '"updated": 1' in run.row_counts_json
|
|
async with factory() as s:
|
|
fixed = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
|
assert fixed.revenue == 207431 # rewritten in place
|
|
assert fixed.accession == stale.accession
|
|
assert fixed.import_run_id == run.id # rewrite is attributable
|
|
assert fixed.import_run_id != first.id
|
|
|
|
|
|
async def test_reparse_leaves_unchanged_rows_untouched(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
first = await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
|
|
|
|
run = await run_import(
|
|
SecFundamentalsImporter(
|
|
client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
|
|
),
|
|
engine=engine,
|
|
force=True,
|
|
)
|
|
|
|
assert '"updated": 0' in run.row_counts_json
|
|
async with factory() as s:
|
|
row = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
|
assert row.import_run_id == first.id # provenance preserved, no needless rewrite
|
|
|
|
|
|
async def test_without_reparse_a_differing_row_stays_immutable(engine, monkeypatch):
|
|
"""The default contract is unchanged: report the discrepancy, never mutate."""
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
await _import_with_old_tolerance(engine, monkeypatch)
|
|
|
|
importer = SecFundamentalsImporter(
|
|
client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
|
|
)
|
|
async with _factory(engine)() as db:
|
|
await importer.detect_revision(db)
|
|
staged = await importer.stage(db)
|
|
importer.reparse = False # same staged diff, default disposition
|
|
counts = await importer.promote(db, staged, run_id=999)
|
|
await db.commit()
|
|
|
|
assert staged.discrepancies, "the diff should still be detected and reported"
|
|
assert counts["updated"] == 0
|
|
async with factory() as s:
|
|
row = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
|
assert row.revenue is None # untouched
|
|
|
|
|
|
async def test_force_bypasses_the_unchanged_revision_no_op(engine):
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
|
|
|
|
same = _importer(_445_client(), today=date(2026, 6, 2))
|
|
assert (await run_import(same, engine=engine)).status == "no_op"
|
|
|
|
forced = SecFundamentalsImporter(
|
|
client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
|
|
)
|
|
assert (await run_import(forced, engine=engine, force=True)).status == STATUS_PROMOTED
|
|
|
|
|
|
# --- CIK resolution: successor registrants with no filings -----------------
|
|
|
|
async def test_issuer_with_no_xbrl_filings_is_reported_not_silent(engine):
|
|
"""XOM resolved to CIK 2115436 'ExxonMobil Holdings Corp', which has zero
|
|
filings, so it produced no snapshots and nothing said why."""
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193},
|
|
companyfacts={320193: _companyfacts([CF_K], [SH_K])},
|
|
submissions={320193: {**_submissions([]), "name": "Shell Holdings Corp"}},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
importer = _importer(client)
|
|
async with factory() as db:
|
|
await importer.detect_revision(db)
|
|
staged = await importer.stage(db)
|
|
result = await importer.validate(db, staged)
|
|
|
|
assert result.summary["no_xbrl_filings_count"] == 1
|
|
assert staged.no_xbrl_filings[0]["cik"] == "0000320193"
|
|
assert staged.no_xbrl_filings[0]["name"] == "Shell Holdings Corp"
|
|
|
|
|
|
async def test_cik_override_pins_a_ticker_to_the_real_filer(engine):
|
|
from app.models.settings import SystemSetting
|
|
from app.services.sec_universe import CIK_OVERRIDES_KEY, resolve_ciks
|
|
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
async with factory() as s:
|
|
s.add(SystemSetting(key=CIK_OVERRIDES_KEY, value='{"AAPL": 34088}'))
|
|
await s.commit()
|
|
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193}, # SEC points at the wrong registrant
|
|
companyfacts={}, submissions={}, latest_index=date(2026, 1, 31),
|
|
)
|
|
async with factory() as db:
|
|
resolved = await resolve_ciks(db, client)
|
|
|
|
assert resolved.symbol_to_cik["AAPL"] == 34088
|
|
assert resolved.cik_updates == [(1, "0000034088")]
|
|
|
|
|
|
async def test_malformed_cik_override_is_ignored_not_fatal(engine):
|
|
from app.models.settings import SystemSetting
|
|
from app.services.sec_universe import CIK_OVERRIDES_KEY, resolve_ciks
|
|
|
|
factory = _factory(engine)
|
|
await _seed(factory, ["AAPL"])
|
|
async with factory() as s:
|
|
s.add(SystemSetting(key=CIK_OVERRIDES_KEY, value="not json at all"))
|
|
await s.commit()
|
|
|
|
client = FakeSecClient(
|
|
tickers={"AAPL": 320193}, companyfacts={}, submissions={},
|
|
latest_index=date(2026, 1, 31),
|
|
)
|
|
async with factory() as db:
|
|
resolved = await resolve_ciks(db, client)
|
|
|
|
assert resolved.symbol_to_cik["AAPL"] == 320193 # fell back to company_tickers
|