Recover SEC facts misfiled under a co-registrant CIK
Deploy / lint (push) Successful in 10s
Deploy / test (push) Successful in 1m47s
Deploy / deploy (push) Successful in 39s

The fundamentals import had been failing for three days on two tracked
filings the index listed but Company Facts appeared not to have. They were
not lagging: SEC filed the XBRL of NEE's and DOW's 2026-07-24 combined
parent/subsidiary 10-Qs under the co-registrant's CIK (Florida Power &
Light, Dow Chemical), so the ticker-carrying filer's own facts file never
receives that accession. This does not self-correct - an NEE filing
misattributed the same way in 2014 is still misfiled.

Because source_max_date advances only on a promoted run, the failure was
self-perpetuating: every later run re-walked the same index day and re-hit
the same two filings.

- Recover from the co-registrant file. The daily index lists every
  co-registrant of an accession, which is the only pointer to where the
  facts actually landed. Rows are re-stamped to the real filer, since
  parse_snapshots stamps the CIK of the payload it read.
- Guard the recovery with a share-count continuity check against the
  issuer's own history, so a subsidiary's standalone facts can never be
  stored as the parent's. No history, no recovery.
- Bound the blocking: a filing still unresolvable after
  MISSING_XBRL_RETRY_DAYS promotes with a named unresolved_filing warning
  instead of wedging every later import.
- Name the offending filings in the alert and record them in
  validation_json, separating not_in_companyfacts from not_in_submissions.
  The gate previously reported a count and discarded the accessions.

Verified against live SEC data: both filings recover with the correct CIK
and the guard rejects a mismatched reference.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-27 11:52:59 +02:00
co-authored by Claude Opus 5
parent d2a27d4a78
commit a7aefa6fe7
2 changed files with 436 additions and 24 deletions
+148 -1
View File
@@ -3,6 +3,7 @@ import framework with a fake SEC client (no network)."""
from __future__ import annotations
import json
import os
import tempfile
from datetime import date, datetime, timezone
@@ -14,6 +15,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_asyn
from app.database import Base
import app.models # noqa: F401
from app.models.fundamental_snapshot import FundamentalSnapshot
from app.models.system_event import SystemEvent
from app.models.ticker import Ticker
from app.services.data_import import STATUS_FAILED, STATUS_PROMOTED, run_import
from app.services.sec_fundamentals_importer import SecFundamentalsImporter
@@ -212,10 +214,155 @@ async def test_consistency_gate_fails_when_facts_lag_index(engine):
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
assert run.status == STATUS_FAILED
assert "Company Facts" in (run.error_details or "")
# The gate blocks every later run until it clears, so the alert itself has to
# name the filing and say why it could not be resolved.
details = run.error_details or ""
assert "GHOST" in details and "not_in_companyfacts" in details
assert "2026-05-01" in details # index date the filing was seen on
summary = json.loads(run.validation_json or "{}")
assert summary["missing_xbrl_count"] == 1
assert summary["missing_xbrl"][0]["accession"] == "GHOST"
assert summary["missing_xbrl"][0]["form"] == "10-Q"
assert await _count(factory, FundamentalSnapshot) == 2 # nothing new written
async def test_gate_separates_missing_submissions_from_missing_facts(engine):
"""An index row the issuer's own filing list does not carry is a different
failure from a Company-Facts lag, and must not be reported as one."""
factory = _factory(engine)
await _seed(factory, ["AAPL"])
backfill = FakeSecClient(
tickers={"AAPL": 320193},
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
submissions={320193: _submissions(SUB_FILINGS)},
latest_index=date(2026, 1, 31),
)
await run_import(_importer(backfill), engine=engine)
incr = FakeSecClient(
tickers={"AAPL": 320193},
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
submissions={320193: _submissions(SUB_FILINGS)}, # submissions never lists ORPHAN
latest_index=date(2026, 5, 2),
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "ORPHAN"}]},
)
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
assert run.status == STATUS_FAILED
assert "ORPHAN" in (run.error_details or "")
assert "not_in_submissions" in (run.error_details or "")
summary = json.loads(run.validation_json or "{}")
assert summary["missing_xbrl"][0]["reason"] == "not_in_submissions"
def _coregistrant_client(share_fact):
"""Incremental client where Q2A's facts landed in co-registrant 99999's file
instead of the filer's own — the NEE-via-FPL / DOW-via-Dow-Chemical shape."""
cf_q2 = _rev("2025-09-28", "2026-03-28", 254940, 2026, "Q2", "Q2A")
return FakeSecClient(
tickers={"AAPL": 320193},
companyfacts={
320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), # filer's own: no Q2A
99999: _companyfacts([cf_q2], [share_fact], cik=99999),
},
submissions={320193: _submissions(SUB_FILINGS + [
_filing("Q2A", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
latest_index=date(2026, 5, 2),
daily={date(2026, 5, 1): [
# one combined filing, listed by the index under both co-registrants
{"form": "10-Q", "cik": 320193, "accession": "Q2A"},
{"form": "10-Q", "cik": 99999, "accession": "Q2A"},
]},
)
async def test_recovers_facts_misfiled_under_coregistrant(engine):
factory = _factory(engine)
await _seed(factory, ["AAPL"])
backfill = FakeSecClient(
tickers={"AAPL": 320193},
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
submissions={320193: _submissions(SUB_FILINGS)},
latest_index=date(2026, 1, 31),
)
await run_import(_importer(backfill), engine=engine)
# 14687 shares is continuous with the issuer's own history (14681 last quarter).
incr = _coregistrant_client(_shares("2026-04-17", 14687, "Q2A", 2026, "Q2"))
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
assert run.status == STATUS_PROMOTED
summary = json.loads(run.validation_json or "{}")
assert summary["recovered_count"] == 1
assert summary["recovered_from_coregistrant"][0]["source_cik"] == "0000099999"
assert summary["missing_xbrl_count"] == 0
async with factory() as s:
q2 = (await s.execute(
select(FundamentalSnapshot).where(FundamentalSnapshot.accession == "Q2A")
)).scalar_one()
codes = (await s.execute(select(SystemEvent.code))).scalars().all()
# Stamped to the issuer that filed, NOT the co-registrant whose file it came from.
assert q2.cik == "0000320193"
assert q2.revenue == 254940 and q2.shares_outstanding == 14687
assert "coregistrant_recovery" in codes
async def test_coregistrant_recovery_rejects_discontinuous_share_count(engine):
"""A co-registrant shell's standalone facts must never be stored as the
parent's — a token float is the signature and it has to be refused."""
factory = _factory(engine)
await _seed(factory, ["AAPL"])
backfill = FakeSecClient(
tickers={"AAPL": 320193},
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
submissions={320193: _submissions(SUB_FILINGS)},
latest_index=date(2026, 1, 31),
)
await run_import(_importer(backfill), engine=engine)
incr = _coregistrant_client(_shares("2026-04-17", 100, "Q2A", 2026, "Q2"))
run = await run_import(_importer(incr, today=date(2026, 5, 3)), engine=engine)
assert run.status == STATUS_FAILED
assert "coregistrant_facts_rejected" in (run.error_details or "")
assert await _count(factory, FundamentalSnapshot) == 2 # nothing recovered
async def test_unresolved_filing_stops_blocking_after_retry_window(engine):
"""A filing SEC has misfiled must not wedge every later import forever."""
factory = _factory(engine)
await _seed(factory, ["AAPL"])
backfill = FakeSecClient(
tickers={"AAPL": 320193},
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])},
submissions={320193: _submissions(SUB_FILINGS)},
latest_index=date(2026, 1, 31),
)
await run_import(_importer(backfill), engine=engine)
incr = FakeSecClient(
tickers={"AAPL": 320193},
companyfacts={320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1])}, # no GHOST
submissions={320193: _submissions(SUB_FILINGS + [
_filing("GHOST", "10-Q", "2026-03-28", "2026-05-01", "2026-05-01T10:01:00.000Z")])},
latest_index=date(2026, 5, 2),
daily={date(2026, 5, 1): [{"form": "10-Q", "cik": 320193, "accession": "GHOST"}]},
)
# 9 days after the index date — well past the retry window.
run = await run_import(_importer(incr, today=date(2026, 5, 10)), engine=engine)
assert run.status == STATUS_PROMOTED # promoted around it, not blocked by it
assert run.source_max_date == date(2026, 5, 2) # and the index advances
summary = json.loads(run.validation_json or "{}")
assert summary["missing_xbrl_count"] == 1 and summary["missing_xbrl_blocking"] == 0
async with factory() as s:
events = (await s.execute(select(SystemEvent))).scalars().all()
unresolved = [e for e in events if e.code == "unresolved_filing"]
assert len(unresolved) == 1 and "GHOST" in unresolved[0].message
async def test_non_xbrl_amendment_skipped_not_failed(engine):
factory = _factory(engine)
await _seed(factory, ["AAPL"])