feat(sec): reparse path, CIK overrides, resolution validation

Operational plumbing to land the parser fixes and to make silent resolution failures visible. - Reparse: SecFundamentalsImporter(reparse=True) restages every accession with the current parser and rewrites the ones that now reconstruct differently, writing the full column set so a row is never half old-parse. Snapshots stay immutable with respect to SEC; the stored row is our reconstruction, and after a parser fix keeping it is a stale cache, not history. run_import(force=True) bypasses the unchanged-revision no-op, since the staleness is on our side, not the source's. Exposed as scripts/reparse_fundamentals.py, dry-run by default. - CIK overrides: sec_universe reads a {symbol: cik} pin from SystemSetting['sec_cik_overrides'], applied ahead of company_tickers.json, for when SEC maps a ticker to a successor shell with no filings (XOM -> a zero- filing "ExxonMobil Holdings Corp" while every 10-K/Q is under CIK 34088). - Resolution validation: a tracked issuer resolving to a registrant with no XBRL filings now records no_xbrl_filings and raises a warning naming the CIKs and the override setting, instead of silently yielding nothing on every run. - _diff_fields compares datetime instants, not representations: accepted_at round-trips naive from SQLite but tz-aware from Postgres, which otherwise made a reparse of identical data report every row as changed (and false-positived the pre-existing discrepancy warning). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-24 10:24:05 +02:00
parent 921f3d06fb
commit e54f03cba6
5 changed files with 449 additions and 12 deletions
@@ -411,3 +411,181 @@ async def test_discrepancy_in_shares_is_detected_and_reported(engine):
        assert k.shares_outstanding == 999.0 and k.import_run_id == 1  # immutable — not overwritten
        events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all()
    assert len(events) == 1 and events[0].severity == "warning"
+
+
+# --- reparse: rewriting rows a fixed parser reconstructs differently --------
+
+# A 4-4-5 filer's YTD-Q3 span (36 weeks = 251 days). The old 20-day tolerance
+# around 273 rejected it and stored revenue=None; 25 accepts it. Reparsing with
+# the fixed parser is exactly the situation this mode exists for.
+CF_Q3_445 = _rev("2025-09-01", "2026-05-10", 207431, 2026, "Q3", "Q3F")
+SUB_445 = [_filing("Q3F", "10-Q", "2026-05-10", "2026-06-01", "2026-06-01T10:01:00.000Z")]
+
+
+def _445_client():
+    return FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_Q3_445], [_shares("2026-05-15", 100, "Q3F", 2026, "Q3")])},
+        submissions={320193: _submissions(SUB_445)},
+        latest_index=date(2026, 6, 1),
+    )
+
+
+async def _import_with_old_tolerance(engine, monkeypatch):
+    """Seed the DB the way the pre-fix parser did: Q3 revenue rejected -> null."""
+    from app.services import sec_facts_parser
+
+    monkeypatch.setattr(sec_facts_parser, "_YTD_TOLERANCE_DAYS", 20)
+    run = await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
+    monkeypatch.undo()
+    return run
+
+
+async def test_reparse_rewrites_rows_the_fixed_parser_reads_differently(engine, monkeypatch):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    first = await _import_with_old_tolerance(engine, monkeypatch)
+
+    async with factory() as s:
+        stale = (await s.execute(select(FundamentalSnapshot))).scalar_one()
+    assert stale.revenue is None, "precondition: the old parser stored a null"
+
+    # Reparse with the current (fixed) parser. force=True because SEC has not
+    # changed -- the staleness is on our side, so the revision gate would no-op.
+    run = await run_import(
+        SecFundamentalsImporter(
+            client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
+        ),
+        engine=engine,
+        force=True,
+    )
+
+    assert run.status == STATUS_PROMOTED
+    assert '"updated": 1' in run.row_counts_json
+    async with factory() as s:
+        fixed = (await s.execute(select(FundamentalSnapshot))).scalar_one()
+    assert fixed.revenue == 207431          # rewritten in place
+    assert fixed.accession == stale.accession
+    assert fixed.import_run_id == run.id    # rewrite is attributable
+    assert fixed.import_run_id != first.id
+
+
+async def test_reparse_leaves_unchanged_rows_untouched(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    first = await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
+
+    run = await run_import(
+        SecFundamentalsImporter(
+            client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
+        ),
+        engine=engine,
+        force=True,
+    )
+
+    assert '"updated": 0' in run.row_counts_json
+    async with factory() as s:
+        row = (await s.execute(select(FundamentalSnapshot))).scalar_one()
+    assert row.import_run_id == first.id  # provenance preserved, no needless rewrite
+
+
+async def test_without_reparse_a_differing_row_stays_immutable(engine, monkeypatch):
+    """The default contract is unchanged: report the discrepancy, never mutate."""
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    await _import_with_old_tolerance(engine, monkeypatch)
+
+    importer = SecFundamentalsImporter(
+        client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
+    )
+    async with _factory(engine)() as db:
+        await importer.detect_revision(db)
+        staged = await importer.stage(db)
+        importer.reparse = False  # same staged diff, default disposition
+        counts = await importer.promote(db, staged, run_id=999)
+        await db.commit()
+
+    assert staged.discrepancies, "the diff should still be detected and reported"
+    assert counts["updated"] == 0
+    async with factory() as s:
+        row = (await s.execute(select(FundamentalSnapshot))).scalar_one()
+    assert row.revenue is None  # untouched
+
+
+async def test_force_bypasses_the_unchanged_revision_no_op(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
+
+    same = _importer(_445_client(), today=date(2026, 6, 2))
+    assert (await run_import(same, engine=engine)).status == "no_op"
+
+    forced = SecFundamentalsImporter(
+        client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
+    )
+    assert (await run_import(forced, engine=engine, force=True)).status == STATUS_PROMOTED
+
+
+# --- CIK resolution: successor registrants with no filings -----------------
+
+async def test_issuer_with_no_xbrl_filings_is_reported_not_silent(engine):
+    """XOM resolved to CIK 2115436 'ExxonMobil Holdings Corp', which has zero
+    filings, so it produced no snapshots and nothing said why."""
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    client = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K], [SH_K])},
+        submissions={320193: {**_submissions([]), "name": "Shell Holdings Corp"}},
+        latest_index=date(2026, 1, 31),
+    )
+    importer = _importer(client)
+    async with factory() as db:
+        await importer.detect_revision(db)
+        staged = await importer.stage(db)
+        result = await importer.validate(db, staged)
+
+    assert result.summary["no_xbrl_filings_count"] == 1
+    assert staged.no_xbrl_filings[0]["cik"] == "0000320193"
+    assert staged.no_xbrl_filings[0]["name"] == "Shell Holdings Corp"
+
+
+async def test_cik_override_pins_a_ticker_to_the_real_filer(engine):
+    from app.models.settings import SystemSetting
+    from app.services.sec_universe import CIK_OVERRIDES_KEY, resolve_ciks
+
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    async with factory() as s:
+        s.add(SystemSetting(key=CIK_OVERRIDES_KEY, value='{"AAPL": 34088}'))
+        await s.commit()
+
+    client = FakeSecClient(
+        tickers={"AAPL": 320193},  # SEC points at the wrong registrant
+        companyfacts={}, submissions={}, latest_index=date(2026, 1, 31),
+    )
+    async with factory() as db:
+        resolved = await resolve_ciks(db, client)
+
+    assert resolved.symbol_to_cik["AAPL"] == 34088
+    assert resolved.cik_updates == [(1, "0000034088")]
+
+
+async def test_malformed_cik_override_is_ignored_not_fatal(engine):
+    from app.models.settings import SystemSetting
+    from app.services.sec_universe import CIK_OVERRIDES_KEY, resolve_ciks
+
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    async with factory() as s:
+        s.add(SystemSetting(key=CIK_OVERRIDES_KEY, value="not json at all"))
+        await s.commit()
+
+    client = FakeSecClient(
+        tickers={"AAPL": 320193}, companyfacts={}, submissions={},
+        latest_index=date(2026, 1, 31),
+    )
+    async with factory() as db:
+        resolved = await resolve_ciks(db, client)
+
+    assert resolved.symbol_to_cik["AAPL"] == 320193  # fell back to company_tickers