fix(sec): A3 sign-off hardening — validate per-concept units structure

- Extend the companyfacts structural check to reject a concept with a
  missing/non-dict `units` mapping (not just the top-level `facts`), so a
  partially-malformed payload fails promotion instead of silently dropping that
  concept's facts. New fixture proves it fails.
- Strengthen the newly-added-issuer test: keep latest_index equal to the prior
  run so ONLY the universe fingerprint changes the revision — proving the
  fingerprint alone prevents a new ticker from being starved/no_op'd.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-22 20:04:11 +02:00
co-authored by Claude Opus 4.8
parent 8dcdcac2a6
commit 96d3b40560
2 changed files with 46 additions and 4 deletions
+18 -2
View File
@@ -166,10 +166,11 @@ class SecFundamentalsImporter:
async def _stage_issuer(self, client, cik, is_backfill, filed_by_cik, staged) -> None:
cf = await client.companyfacts(cik)
if not isinstance(cf, dict) or not isinstance(cf.get("facts"), dict):
bad = _companyfacts_structure_error(cf)
if bad is not None:
# Malformed payload (missing facts/units structure) — record separately
# and fail validation, rather than letting it degrade to skipped rows.
staged.invalid_payloads.append({"cik": cik10(cik), "reason": "missing facts structure"})
staged.invalid_payloads.append({"cik": cik10(cik), "reason": bad})
staged.issuers_fetched += 1
return
sub = await client.submissions(cik, include_history=is_backfill)
@@ -345,6 +346,21 @@ class SecFundamentalsImporter:
return {r.accession: r for r in rows}
def _companyfacts_structure_error(cf: Any) -> str | None:
"""None if the payload is structurally sound, else a reason string. Checks the
top-level ``facts`` mapping AND that every concept carries a ``units`` mapping —
a missing/non-dict units would silently drop that concept's facts otherwise."""
if not isinstance(cf, dict) or not isinstance(cf.get("facts"), dict):
return "missing facts structure"
for concepts in cf["facts"].values():
if not isinstance(concepts, dict):
return "malformed taxonomy structure"
for body in concepts.values():
if not isinstance(body, dict) or not isinstance(body.get("units"), dict):
return "missing units structure"
return None
def _filing_meta(sub: dict[str, Any]) -> tuple[dict[str, FilingMeta], set[str]]:
"""(xbrl_meta, nonxbrl_accessions) from a submissions payload. xbrl_meta only
includes 10-K/10-Q(/A) filings that are XBRL and have full period metadata."""