Recover SEC facts misfiled under a co-registrant CIK

The fundamentals import had been failing for three days on two tracked filings the index listed but Company Facts appeared not to have. They were not lagging: SEC filed the XBRL of NEE's and DOW's 2026-07-24 combined parent/subsidiary 10-Qs under the co-registrant's CIK (Florida Power & Light, Dow Chemical), so the ticker-carrying filer's own facts file never receives that accession. This does not self-correct - an NEE filing misattributed the same way in 2014 is still misfiled. Because source_max_date advances only on a promoted run, the failure was self-perpetuating: every later run re-walked the same index day and re-hit the same two filings. - Recover from the co-registrant file. The daily index lists every co-registrant of an accession, which is the only pointer to where the facts actually landed. Rows are re-stamped to the real filer, since parse_snapshots stamps the CIK of the payload it read. - Guard the recovery with a share-count continuity check against the issuer's own history, so a subsidiary's standalone facts can never be stored as the parent's. No history, no recovery. - Bound the blocking: a filing still unresolvable after MISSING_XBRL_RETRY_DAYS promotes with a named unresolved_filing warning instead of wedging every later import. - Name the offending filings in the alert and record them in validation_json, separating not_in_companyfacts from not_in_submissions. The gate previously reported a count and discarded the accessions. Verified against live SEC data: both filings recover with the correct CIK and the guard rejects a mismatched reference. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-07-27 11:52:59 +02:00
parent d2a27d4a78
commit a7aefa6fe7
2 changed files with 436 additions and 24 deletions
@@ -3,6 +3,7 @@ import framework with a fake SEC client (no network)."""

 from __future__ import annotations

+import json
 import os
 import tempfile
 from datetime import date, datetime, timezone
@@ -14,6 +15,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn
 from app.database import Base
 import app.models  # noqa: F401
 from app.models.fundamental_snapshot import FundamentalSnapshot
+from app.models.system_event import SystemEvent
 from app.models.ticker import Ticker
 from app.services.data_import import STATUS_FAILED, STATUS_PROMOTED, run_import
 from app.services.sec_fundamentals_importer import SecFundamentalsImporter
@@ -212,10 +214,155 @@ async def test_consistency_gate_fails_when_facts_lag_index(engine):
    run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)

    assert run.status == STATUS_FAILED
-    assert "Company Facts" in (run.error_details or "")
+    # The gate blocks every later run until it clears, so the alert itself has to
+    # name the filing and say why it could not be resolved.
+    details = run.error_details or ""
+    assert "GHOST" in details and "not_in_companyfacts" in details
+    assert "2026-05-01" in details  # index date the filing was seen on
+    summary = json.loads(run.validation_json or "{}")
+    assert summary["missing_xbrl_count"] == 1
+    assert summary["missing_xbrl"][0]["accession"] == "GHOST"
+    assert summary["missing_xbrl"][0]["form"] == "10-Q"
    assert await _count(factory, FundamentalSnapshot) == 2  # nothing new written


+async def test_gate_separates_missing_submissions_from_missing_facts(engine):
+    """An index row the issuer's own filing list does not carry is a different
+    failure from a Company-Facts lag, and must not be reported as one."""
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    backfill = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},
+        latest_index=date(2026, 1, 31),
+    )
+    await run_import(_importer(backfill), engine=engine)
+
+    incr = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},  # submissions never lists ORPHAN
+        latest_index=date(2026, 5, 2),
+        daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "ORPHAN"}]},
+    )
+    run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
+
+    assert run.status == STATUS_FAILED
+    assert "ORPHAN" in (run.error_details or "")
+    assert "not_in_submissions" in (run.error_details or "")
+    summary = json.loads(run.validation_json or "{}")
+    assert summary["missing_xbrl"][0]["reason"] == "not_in_submissions"
+
+
+def _coregistrant_client(share_fact):
+    """Incremental client where Q2A's facts landed in co-registrant 99999's file
+    instead of the filer's own — the NEE-via-FPL / DOW-via-Dow-Chemical shape."""
+    cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A")
+    return FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={
+            320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),  # filer's own: no Q2A
+            99999: _companyfacts([cf_q2], [share_fact], cik=99999),
+        },
+        submissions={320193: _submissions(SUB_FILINGS + [
+            _filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
+        latest_index=date(2026, 5, 2),
+        daily={date(2026, 5, 1): [
+            # one combined filing, listed by the index under both co-registrants
+            {"form": "10-Q", "cik": 320193, "accession": "Q2A"},
+            {"form": "10-Q", "cik": 99999, "accession": "Q2A"},
+        ]},
+    )
+
+
+async def test_recovers_facts_misfiled_under_coregistrant(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    backfill = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},
+        latest_index=date(2026, 1, 31),
+    )
+    await run_import(_importer(backfill), engine=engine)
+
+    # 14687 shares is continuous with the issuer's own history (14681 last quarter).
+    incr = _coregistrant_client(_shares("2026-04-17", 14687, "Q2A", 2026, "Q2"))
+    run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
+
+    assert run.status == STATUS_PROMOTED
+    summary = json.loads(run.validation_json or "{}")
+    assert summary["recovered_count"] == 1
+    assert summary["recovered_from_coregistrant"][0]["source_cik"] == "0000099999"
+    assert summary["missing_xbrl_count"] == 0
+
+    async with factory() as s:
+        q2 = (await s.execute(
+            select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A")
+        )).scalar_one()
+        codes = (await s.execute(select(SystemEvent.code))).scalars().all()
+    # Stamped to the issuer that filed, NOT the co-registrant whose file it came from.
+    assert q2.cik == "0000320193"
+    assert q2.revenue == 254940 and q2.shares_outstanding == 14687
+    assert "coregistrant_recovery" in codes
+
+
+async def test_coregistrant_recovery_rejects_discontinuous_share_count(engine):
+    """A co-registrant shell's standalone facts must never be stored as the
+    parent's — a token float is the signature and it has to be refused."""
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    backfill = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},
+        latest_index=date(2026, 1, 31),
+    )
+    await run_import(_importer(backfill), engine=engine)
+
+    incr = _coregistrant_client(_shares("2026-04-17", 100, "Q2A", 2026, "Q2"))
+    run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
+
+    assert run.status == STATUS_FAILED
+    assert "coregistrant_facts_rejected" in (run.error_details or "")
+    assert await _count(factory, FundamentalSnapshot) == 2  # nothing recovered
+
+
+async def test_unresolved_filing_stops_blocking_after_retry_window(engine):
+    """A filing SEC has misfiled must not wedge every later import forever."""
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL"])
+    backfill = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
+        submissions={320193: _submissions(SUB_FILINGS)},
+        latest_index=date(2026, 1, 31),
+    )
+    await run_import(_importer(backfill), engine=engine)
+
+    incr = FakeSecClient(
+        tickers={"AAPL": 320193},
+        companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},  # no GHOST
+        submissions={320193: _submissions(SUB_FILINGS + [
+            _filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
+        latest_index=date(2026, 5, 2),
+        daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]},
+    )
+    # 9 days after the index date — well past the retry window.
+    run = await run_import(_importer(incr, today=date(2026, 5, 10)), engine=engine)
+
+    assert run.status == STATUS_PROMOTED  # promoted around it, not blocked by it
+    assert run.source_max_date == date(2026, 5, 2)  # and the index advances
+    summary = json.loads(run.validation_json or "{}")
+    assert summary["missing_xbrl_count"] == 1 and summary["missing_xbrl_blocking"] == 0
+
+    async with factory() as s:
+        events = (await s.execute(select(SystemEvent))).scalars().all()
+    unresolved = [e for e in events if e.code == "unresolved_filing"]
+    assert len(unresolved) == 1 and "GHOST" in unresolved[0].message
+
+
 async def test_non_xbrl_amendment_skipped_not_failed(engine):
    factory = _factory(engine)
    await _seed(factory, ["AAPL"])