"""Integration tests for the SEC fundamentals importer, driven through the real import framework with a fake SEC client (no network).""" from __future__ import annotations import json import os import tempfile from datetime import date, datetime, timezone import pytest from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from app.database import Base import app.models # noqa: F401 from app.models.fundamental_snapshot import FundamentalSnapshot from app.models.system_event import SystemEvent from app.models.ticker import Ticker from app.services.data_import import ( STATUS_DEFERRED, STATUS_FAILED, STATUS_PROMOTED, run_import, ) from app.services.sec_fundamentals_importer import SecFundamentalsImporter @pytest.fixture async def engine(): fd, path = tempfile.mkstemp(suffix=".db") os.close(fd) eng = create_async_engine(f"sqlite+aiosqlite:///{path}") async with eng.begin() as conn: await conn.run_sync(Base.metadata.create_all) try: yield eng finally: await eng.dispose() try: os.unlink(path) except OSError: pass def _factory(engine): return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) # --- fixture SEC data (AAPL, cik 320193) ----------------------------------- def _rev(start, end, val, fy, fp, accn): return {"start": start, "end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"} def _shares(end, val, accn, fy, fp): return {"end": end, "val": val, "fy": fy, "fp": fp, "accn": accn, "form": "10-K"} def _companyfacts(rev_facts, share_facts, cik=320193): return { "cik": cik, "facts": { "us-gaap": {"RevenueFromContractWithCustomerExcludingAssessedTax": {"units": {"USD": rev_facts}}}, "dei": {"EntityCommonStockSharesOutstanding": {"units": {"shares": share_facts}}}, }, } def _filing(accn, form, report, filed, accepted, is_xbrl=True): return {"accession": accn, "form": form, "report_date": report, "filing_date": filed, "acceptance_datetime": accepted, "is_xbrl": is_xbrl} CF_K = _rev("2024-09-29", "2025-09-27", 416161, 2025, "FY", "K") CF_Q1 = _rev("2025-09-28", "2025-12-27", 143756, 2026, "Q1", "Q") SH_K = _shares("2025-10-17", 14776, "K", 2025, "FY") SH_Q1 = _shares("2026-01-16", 14681, "Q", 2026, "Q1") SUB_FILINGS = [ _filing("K", "10-K", "2025-09-27", "2025-10-31", "2025-10-31T10:01:26.000Z"), _filing("Q", "10-Q", "2025-12-27", "2026-01-30", "2026-01-30T11:01:00.000Z"), ] def _submissions(filings): return {"cik": 320193, "sic": "3571", "sic_description": "Electronic Computers", "fiscal_year_end": "0926", "tickers": ["AAPL"], "filings": filings} class FakeSecClient: def __init__(self, *, tickers, companyfacts, submissions, latest_index, daily=None): self._tickers = tickers self._cf = companyfacts self._sub = submissions self._latest = latest_index self._daily = daily or {} async def __aenter__(self): return self async def __aexit__(self, *a): return False async def company_tickers(self): return dict(self._tickers) async def latest_index_date(self, today=None): return self._latest async def daily_index(self, day): return list(self._daily.get(day, [])) async def companyfacts(self, cik): return self._cf[int(cik)] async def submissions(self, cik, *, include_history=False): return self._sub[int(cik)] def _importer(client, today=date(2026, 2, 1)): return SecFundamentalsImporter(client_factory=lambda: client, today=today) async def _seed(factory, symbols): async with factory() as s: for sym in symbols: s.add(Ticker(symbol=sym)) await s.commit() async def _count(factory, model): async with factory() as s: return (await s.execute(select(func.count()).select_from(model))).scalar_one() # --------------------------------------------------------------------------- async def test_backfill_inserts_snapshots_and_ticker_meta(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) client = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) run = await run_import(_importer(client), engine=engine) assert run.status == STATUS_PROMOTED assert run.source_max_date == date(2026, 1, 31) assert await _count(factory, FundamentalSnapshot) == 2 async with factory() as s: t = (await s.execute(select(Ticker))).scalar_one() assert t.cik == "0000320193" and t.sic == "3571" snaps = (await s.execute(select(FundamentalSnapshot))).scalars().all() assert {x.fiscal_period for x in snaps} == {"FY", "Q1"} assert all(x.import_run_id == run.id for x in snaps) fy = next(x for x in snaps if x.fiscal_period == "FY") assert fy.revenue == 416161 and fy.shares_outstanding == 14776 async def test_incremental_adds_only_new_filing(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) assert await _count(factory, FundamentalSnapshot) == 2 # A new Q2 10-Q appears in the daily index and Company Facts. cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A") sh_q2 = _shares("2026-04-17", 14687, "Q2A", 2026, "Q2") incr = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])}, submissions={320193: _submissions(SUB_FILINGS + [ _filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])}, latest_index=date(2026, 5, 2), daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "Q2A"}]}, ) run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine) assert run.status == STATUS_PROMOTED assert await _count(factory, FundamentalSnapshot) == 3 # only Q2A added async with factory() as s: q2 = (await s.execute( select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A") )).scalar_one() assert q2.fiscal_period == "Q2" and q2.revenue == 254940 async def test_consistency_gate_defers_without_alert_when_facts_lag_index(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) # Index + submissions list an XBRL filing "GHOST" that Company Facts lacks. incr = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # no GHOST submissions={320193: _submissions(SUB_FILINGS + [ _filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])}, latest_index=date(2026, 5, 2), daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]}, ) run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine) assert run.status == STATUS_DEFERRED # The gate blocks every later run until it clears, so run history still has # to name the filing and say why it could not be resolved. details = run.error_details or "" assert "GHOST" in details and "not_in_companyfacts" in details assert "2026-05-01" in details # index date the filing was seen on summary = json.loads(run.validation_json or "{}") assert summary["missing_xbrl_count"] == 1 assert summary["missing_xbrl"][0]["accession"] == "GHOST" assert summary["missing_xbrl"][0]["form"] == "10-Q" assert await _count(factory, FundamentalSnapshot) == 2 # nothing new written assert await _count(factory, SystemEvent) == 0 # expected SEC lag does not alert async def test_gate_separates_missing_submissions_from_missing_facts(engine): """An index row the issuer's own filing list does not carry is a different failure from a Company-Facts lag, and must not be reported as one.""" factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) incr = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, # submissions never lists ORPHAN latest_index=date(2026, 5, 2), daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "ORPHAN"}]}, ) run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine) assert run.status == STATUS_FAILED assert "ORPHAN" in (run.error_details or "") assert "not_in_submissions" in (run.error_details or "") summary = json.loads(run.validation_json or "{}") assert summary["missing_xbrl"][0]["reason"] == "not_in_submissions" def _coregistrant_client(share_fact): """Incremental client where Q2A's facts landed in co-registrant 99999's file instead of the filer's own — the NEE-via-FPL / DOW-via-Dow-Chemical shape.""" cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A") return FakeSecClient( tickers={"AAPL": 320193}, companyfacts={ 320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), # filer's own: no Q2A 99999: _companyfacts([cf_q2], [share_fact], cik=99999), }, submissions={320193: _submissions(SUB_FILINGS + [ _filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])}, latest_index=date(2026, 5, 2), daily={date(2026, 5, 1): [ # one combined filing, listed by the index under both co-registrants {"form": "10-Q", "cik": 320193, "accession": "Q2A"}, {"form": "10-Q", "cik": 99999, "accession": "Q2A"}, ]}, ) async def test_recovers_facts_misfiled_under_coregistrant(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) # 14687 shares is continuous with the issuer's own history (14681 last quarter). incr = _coregistrant_client(_shares("2026-04-17", 14687, "Q2A", 2026, "Q2")) run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine) assert run.status == STATUS_PROMOTED summary = json.loads(run.validation_json or "{}") assert summary["recovered_count"] == 1 assert summary["recovered_from_coregistrant"][0]["source_cik"] == "0000099999" assert summary["missing_xbrl_count"] == 0 async with factory() as s: q2 = (await s.execute( select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A") )).scalar_one() codes = (await s.execute(select(SystemEvent.code))).scalars().all() # Stamped to the issuer that filed, NOT the co-registrant whose file it came from. assert q2.cik == "0000320193" assert q2.revenue == 254940 and q2.shares_outstanding == 14687 assert "coregistrant_recovery" in codes async def test_coregistrant_recovery_rejects_discontinuous_share_count(engine): """A co-registrant shell's standalone facts must never be stored as the parent's — a token float is the signature and it has to be refused.""" factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) incr = _coregistrant_client(_shares("2026-04-17", 100, "Q2A", 2026, "Q2")) run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine) assert run.status == STATUS_FAILED assert "coregistrant_facts_rejected" in (run.error_details or "") assert await _count(factory, FundamentalSnapshot) == 2 # nothing recovered async def test_unresolved_filing_stops_blocking_after_retry_window(engine): """A filing SEC has misfiled must not wedge every later import forever.""" factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) incr = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # no GHOST submissions={320193: _submissions(SUB_FILINGS + [ _filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])}, latest_index=date(2026, 5, 2), daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]}, ) # 9 days after the index date — well past the retry window. run = await run_import(_importer(incr, today=date(2026, 5, 10)), engine=engine) assert run.status == STATUS_PROMOTED # promoted around it, not blocked by it assert run.source_max_date == date(2026, 5, 2) # and the index advances summary = json.loads(run.validation_json or "{}") assert summary["missing_xbrl_count"] == 1 and summary["missing_xbrl_blocking"] == 0 async with factory() as s: events = (await s.execute(select(SystemEvent))).scalars().all() unresolved = [e for e in events if e.code == "unresolved_filing"] assert len(unresolved) == 1 and "GHOST" in unresolved[0].message async def test_non_xbrl_amendment_skipped_not_failed(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) incr = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS + [ _filing("AMD", "10-K/A", "2025-09-27", "2026-05-01", "2026-05-01T10:01:00.000Z", is_xbrl=False)])}, latest_index=date(2026, 5, 2), daily={date(2026, 5, 1): [{"form": "10-K/A", "cik": 320193, "accession": "AMD"}]}, ) run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine) assert run.status == STATUS_PROMOTED # non-XBRL amendment is skipped, not a failure assert "skipped_non_xbrl" in (run.validation_json or "") assert await _count(factory, FundamentalSnapshot) == 2 async def test_failed_backfill_leaves_tickers_unwritten(engine): factory = _factory(engine) await _seed(factory, ["AAPL", "MSFT", "NVDA"]) # 3 resolve, only AAPL yields rows client = FakeSecClient( tickers={"AAPL": 320193, "MSFT": 789019, "NVDA": 1045810}, companyfacts={ 320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), 789019: {"cik": 789019, "facts": {}}, # no facts -> no rows 1045810: {"cik": 1045810, "facts": {}}, }, submissions={ 320193: _submissions(SUB_FILINGS), 789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []}, 1045810: {"cik": 1045810, "sic": "3674", "sic_description": "y", "filings": []}, }, latest_index=date(2026, 1, 31), ) run = await run_import(_importer(client), engine=engine) assert run.status == STATUS_FAILED # coverage 1/3 < 50% assert "coverage" in (run.error_details or "") assert await _count(factory, FundamentalSnapshot) == 0 # read-only resolution: no ticker cik/sic written on a failed run async with factory() as s: assert all(t.cik is None and t.sic is None for t in (await s.execute(select(Ticker))).scalars()) async def test_index_gap_over_45_days_loses_no_filings(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) # 74-day gap; the filing sits in the OLD part (>45d before latest). cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "OLD") sh_q2 = _shares("2026-04-17", 14687, "OLD", 2026, "Q2") incr = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1, cf_q2], [SH_K, SH_Q1, sh_q2])}, submissions={320193: _submissions(SUB_FILINGS + [ _filing("OLD", "10-Q", "2026-03-28", "2026-02-10", "2026-02-10T10:01:00.000Z")])}, latest_index=date(2026, 4, 15), daily={date(2026, 2, 10): [{"form": "10-Q", "cik": 320193, "accession": "OLD"}]}, ) run = await run_import(_importer(incr, today=date(2026, 4, 16)), engine=engine) assert run.status == STATUS_PROMOTED assert await _count(factory, FundamentalSnapshot) == 3 # the old-gap filing was NOT lost async def test_newly_added_issuer_backfills_without_filing(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) backfill = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) await run_import(_importer(backfill), engine=engine) # MSFT added to the universe later; it did NOT file (not in the daily index). await _seed(factory, ["MSFT"]) msft_rev = _rev("2024-07-01", "2025-06-30", 270000, 2025, "FY", "M") msft_sh = _shares("2025-07-15", 7400, "M", 2025, "FY") incr = FakeSecClient( tickers={"AAPL": 320193, "MSFT": 789019}, companyfacts={ 320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), 789019: _companyfacts([msft_rev], [msft_sh], cik=789019), }, submissions={ 320193: _submissions(SUB_FILINGS), 789019: {"cik": 789019, "sic": "7372", "sic_description": "Prepackaged Software", "filings": [_filing("M", "10-K", "2025-06-30", "2025-07-30", "2025-07-30T10:00:00.000Z")]}, }, # SAME index date as the prior run and no filing: only the universe # fingerprint (MSFT added) changes the revision, so this proves the # fingerprint alone prevents starvation. latest_index=date(2026, 1, 31), daily={}, # MSFT did not file ) run = await run_import(_importer(incr, today=date(2026, 2, 1)), engine=engine) assert run.status == STATUS_PROMOTED async with factory() as s: msft = (await s.execute( select(FundamentalSnapshot).where(FundamentalSnapshot.cik == "0000789019") )).scalars().all() assert len(msft) == 1 and msft[0].revenue == 270000 # full-history backfill despite no filing async def test_malformed_companyfacts_fails_validation(engine): factory = _factory(engine) await _seed(factory, ["AAPL", "MSFT"]) client = FakeSecClient( tickers={"AAPL": 320193, "MSFT": 789019}, companyfacts={ 320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), 789019: {"cik": 789019}, # malformed — no "facts" structure }, submissions={ 320193: _submissions(SUB_FILINGS), 789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []}, }, latest_index=date(2026, 1, 31), ) run = await run_import(_importer(client), engine=engine) assert run.status == STATUS_FAILED assert "malformed" in (run.error_details or "") assert await _count(factory, FundamentalSnapshot) == 0 # nothing promoted async def test_missing_units_structure_fails_validation(engine): factory = _factory(engine) await _seed(factory, ["AAPL", "MSFT"]) client = FakeSecClient( tickers={"AAPL": 320193, "MSFT": 789019}, companyfacts={ 320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), # facts present, but a concept is missing its units mapping 789019: {"cik": 789019, "facts": {"us-gaap": {"Revenues": {"label": "Revenues"}}}}, }, submissions={ 320193: _submissions(SUB_FILINGS), 789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []}, }, latest_index=date(2026, 1, 31), ) run = await run_import(_importer(client), engine=engine) assert run.status == STATUS_FAILED assert "malformed" in (run.error_details or "") assert "units" in (run.validation_json or "") assert await _count(factory, FundamentalSnapshot) == 0 async def test_discrepancy_in_shares_is_detected_and_reported(engine): from app.models.system_event import SystemEvent utc = timezone.utc factory = _factory(engine) await _seed(factory, ["AAPL"]) # Pre-store accession K matching what the parser will produce EXCEPT shares. async with factory() as s: s.add(FundamentalSnapshot( cik="0000320193", accession="K", form="10-K", filed_date=date(2025, 10, 31), accepted_at=datetime(2025, 10, 31, 10, 1, 26, tzinfo=utc), period_start=date(2024, 9, 29), period_end=date(2025, 9, 27), fiscal_year=2025, fiscal_period="FY", revenue=416161.0, shares_outstanding=999.0, import_run_id=1, created_at=datetime(2025, 10, 31, tzinfo=utc))) await s.commit() client = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # SH_K = 14776 != 999 submissions={320193: _submissions(SUB_FILINGS)}, latest_index=date(2026, 1, 31), ) run = await run_import(_importer(client), engine=engine) assert run.status == STATUS_PROMOTED # a discrepancy is reported, not a failure assert '"discrepancy_count": 1' in (run.validation_json or "") assert "shares_outstanding" in (run.validation_json or "") async with factory() as s: k = (await s.execute(select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "K"))).scalar_one() assert k.shares_outstanding == 999.0 and k.import_run_id == 1 # immutable — not overwritten events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all() assert len(events) == 1 and events[0].severity == "warning" # --- reparse: rewriting rows a fixed parser reconstructs differently -------- # A 4-4-5 filer's YTD-Q3 span (36 weeks = 251 days). The old 20-day tolerance # around 273 rejected it and stored revenue=None; 25 accepts it. Reparsing with # the fixed parser is exactly the situation this mode exists for. CF_Q3_445 = _rev("2025-09-01", "2026-05-10", 207431, 2026, "Q3", "Q3F") SUB_445 = [_filing("Q3F", "10-Q", "2026-05-10", "2026-06-01", "2026-06-01T10:01:00.000Z")] def _445_client(): return FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_Q3_445], [_shares("2026-05-15", 100, "Q3F", 2026, "Q3")])}, submissions={320193: _submissions(SUB_445)}, latest_index=date(2026, 6, 1), ) async def _import_with_old_tolerance(engine, monkeypatch): """Seed the DB the way the pre-fix parser did: Q3 revenue rejected -> null.""" from app.services import sec_facts_parser monkeypatch.setattr(sec_facts_parser, "_YTD_TOLERANCE_DAYS", 20) run = await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine) monkeypatch.undo() return run async def test_reparse_rewrites_rows_the_fixed_parser_reads_differently(engine, monkeypatch): factory = _factory(engine) await _seed(factory, ["AAPL"]) first = await _import_with_old_tolerance(engine, monkeypatch) async with factory() as s: stale = (await s.execute(select(FundamentalSnapshot))).scalar_one() assert stale.revenue is None, "precondition: the old parser stored a null" # Reparse with the current (fixed) parser. force=True because SEC has not # changed -- the staleness is on our side, so the revision gate would no-op. run = await run_import( SecFundamentalsImporter( client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True ), engine=engine, force=True, ) assert run.status == STATUS_PROMOTED assert '"updated": 1' in run.row_counts_json async with factory() as s: fixed = (await s.execute(select(FundamentalSnapshot))).scalar_one() assert fixed.revenue == 207431 # rewritten in place assert fixed.accession == stale.accession assert fixed.import_run_id == run.id # rewrite is attributable assert fixed.import_run_id != first.id async def test_reparse_leaves_unchanged_rows_untouched(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) first = await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine) run = await run_import( SecFundamentalsImporter( client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True ), engine=engine, force=True, ) assert '"updated": 0' in run.row_counts_json async with factory() as s: row = (await s.execute(select(FundamentalSnapshot))).scalar_one() assert row.import_run_id == first.id # provenance preserved, no needless rewrite async def test_without_reparse_a_differing_row_stays_immutable(engine, monkeypatch): """The default contract is unchanged: report the discrepancy, never mutate.""" factory = _factory(engine) await _seed(factory, ["AAPL"]) await _import_with_old_tolerance(engine, monkeypatch) importer = SecFundamentalsImporter( client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True ) async with _factory(engine)() as db: await importer.detect_revision(db) staged = await importer.stage(db) importer.reparse = False # same staged diff, default disposition counts = await importer.promote(db, staged, run_id=999) await db.commit() assert staged.discrepancies, "the diff should still be detected and reported" assert counts["updated"] == 0 async with factory() as s: row = (await s.execute(select(FundamentalSnapshot))).scalar_one() assert row.revenue is None # untouched async def test_force_bypasses_the_unchanged_revision_no_op(engine): factory = _factory(engine) await _seed(factory, ["AAPL"]) await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine) same = _importer(_445_client(), today=date(2026, 6, 2)) assert (await run_import(same, engine=engine)).status == "no_op" forced = SecFundamentalsImporter( client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True ) assert (await run_import(forced, engine=engine, force=True)).status == STATUS_PROMOTED # --- CIK resolution: successor registrants with no filings ----------------- async def test_issuer_with_no_xbrl_filings_is_reported_not_silent(engine): """XOM resolved to CIK 2115436 'ExxonMobil Holdings Corp', which has zero filings, so it produced no snapshots and nothing said why.""" factory = _factory(engine) await _seed(factory, ["AAPL"]) client = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={320193: _companyfacts([CF_K], [SH_K])}, submissions={320193: {**_submissions([]), "name": "Shell Holdings Corp"}}, latest_index=date(2026, 1, 31), ) importer = _importer(client) async with factory() as db: await importer.detect_revision(db) staged = await importer.stage(db) result = await importer.validate(db, staged) assert result.summary["no_xbrl_filings_count"] == 1 assert staged.no_xbrl_filings[0]["cik"] == "0000320193" assert staged.no_xbrl_filings[0]["name"] == "Shell Holdings Corp" async def test_cik_override_pins_a_ticker_to_the_real_filer(engine): from app.models.settings import SystemSetting from app.services.sec_universe import CIK_OVERRIDES_KEY, resolve_ciks factory = _factory(engine) await _seed(factory, ["AAPL"]) async with factory() as s: s.add(SystemSetting(key=CIK_OVERRIDES_KEY, value='{"AAPL": 34088}')) await s.commit() client = FakeSecClient( tickers={"AAPL": 320193}, # SEC points at the wrong registrant companyfacts={}, submissions={}, latest_index=date(2026, 1, 31), ) async with factory() as db: resolved = await resolve_ciks(db, client) assert resolved.symbol_to_cik["AAPL"] == 34088 assert resolved.cik_updates == [(1, "0000034088")] async def test_malformed_cik_override_is_ignored_not_fatal(engine): from app.models.settings import SystemSetting from app.services.sec_universe import CIK_OVERRIDES_KEY, resolve_ciks factory = _factory(engine) await _seed(factory, ["AAPL"]) async with factory() as s: s.add(SystemSetting(key=CIK_OVERRIDES_KEY, value="not json at all")) await s.commit() client = FakeSecClient( tickers={"AAPL": 320193}, companyfacts={}, submissions={}, latest_index=date(2026, 1, 31), ) async with factory() as db: resolved = await resolve_ciks(db, client) assert resolved.symbol_to_cik["AAPL"] == 320193 # fell back to company_tickers