feat(sec): reparse path, CIK overrides, resolution validation

Operational plumbing to land the parser fixes and to make silent resolution
failures visible.

- Reparse: SecFundamentalsImporter(reparse=True) restages every accession with
  the current parser and rewrites the ones that now reconstruct differently,
  writing the full column set so a row is never half old-parse. Snapshots stay
  immutable with respect to SEC; the stored row is our reconstruction, and after
  a parser fix keeping it is a stale cache, not history. run_import(force=True)
  bypasses the unchanged-revision no-op, since the staleness is on our side, not
  the source's. Exposed as scripts/reparse_fundamentals.py, dry-run by default.
- CIK overrides: sec_universe reads a {symbol: cik} pin from
  SystemSetting['sec_cik_overrides'], applied ahead of company_tickers.json, for
  when SEC maps a ticker to a successor shell with no filings (XOM -> a zero-
  filing "ExxonMobil Holdings Corp" while every 10-K/Q is under CIK 34088).
- Resolution validation: a tracked issuer resolving to a registrant with no XBRL
  filings now records no_xbrl_filings and raises a warning naming the CIKs and
  the override setting, instead of silently yielding nothing on every run.
- _diff_fields compares datetime instants, not representations: accepted_at
  round-trips naive from SQLite but tz-aware from Postgres, which otherwise made
  a reparse of identical data report every row as changed (and false-positived
  the pre-existing discrepancy warning).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
2026-07-24 10:24:05 +02:00
co-authored by Claude Opus 4.8
parent 921f3d06fb
commit e54f03cba6
5 changed files with 449 additions and 12 deletions
+41 -1
View File
@@ -16,6 +16,7 @@ changes on the framework's failure commit). The proposals are applied only in
from __future__ import annotations
import hashlib
import json
import logging
from dataclasses import dataclass, field
from typing import Iterable
@@ -23,11 +24,21 @@ from typing import Iterable
from sqlalchemy import select, update
from app.models.ticker import Ticker
from app.services import settings_store
from app.services.earnings_alignment import normalise_symbol
from app.services.sec_client import SecClient
logger = logging.getLogger(__name__)
# JSON {symbol: cik} pinning a ticker to a specific registrant, overriding
# company_tickers.json. Needed when SEC maps a ticker to a successor entity that
# has not filed: XOM points at CIK 2115436 "ExxonMobil Holdings Corp" (zero XBRL
# filings) while every 10-K/10-Q — including one filed 2026-05-04 — is still under
# CIK 34088. Which registrant is the real filer is a judgement about a corporate
# event, so it is pinned explicitly rather than guessed. The importer's
# `no_xbrl_filings` warning is what tells you a pin is needed.
CIK_OVERRIDES_KEY = "sec_cik_overrides"
@dataclass
class ResolvedUniverse:
@@ -43,6 +54,7 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
"""Resolve tracked tickers to CIKs via company_tickers.json. **Read-only** —
returns the mapping + proposed `tickers.cik` writes; mutates nothing."""
ticker_to_cik = await client.company_tickers()
overrides = await cik_overrides(db)
rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all()
result = ResolvedUniverse()
@@ -50,7 +62,7 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
if not symbol:
continue
sym = normalise_symbol(symbol)
cik = ticker_to_cik.get(sym)
cik = overrides.get(sym) or ticker_to_cik.get(sym)
if cik is None:
continue # ADRs / non-SEC issuers — snapshots simply absent
result.symbol_to_cik[sym] = cik
@@ -65,6 +77,34 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
return result
async def cik_overrides(db) -> dict[str, int]:
"""Manual ``{symbol: cik}`` pins from ``SystemSetting[CIK_OVERRIDES_KEY]``.
A malformed setting must never take the importer down, so anything unparseable
is logged and ignored — the run then falls back to company_tickers.json.
"""
raw = await settings_store.get_value(db, CIK_OVERRIDES_KEY)
if not raw:
return {}
try:
loaded = json.loads(raw)
except (TypeError, ValueError):
logger.warning("%s is not valid JSON — ignoring CIK overrides", CIK_OVERRIDES_KEY)
return {}
if not isinstance(loaded, dict):
logger.warning("%s must be a {symbol: cik} object — ignoring", CIK_OVERRIDES_KEY)
return {}
out: dict[str, int] = {}
for symbol, cik in loaded.items():
try:
out[normalise_symbol(str(symbol))] = int(cik)
except (TypeError, ValueError):
logger.warning("%s: bad entry %r -> %r — ignoring", CIK_OVERRIDES_KEY, symbol, cik)
if out:
logger.info("resolve_ciks: %d CIK override(s) applied: %s", len(out), sorted(out))
return out
async def fetch_sic_updates(
client: SecClient, cik_to_ticker_ids: dict[int, Iterable[int]]
) -> list[tuple[int, str | None, str | None]]: