feat(sec): reparse path, CIK overrides, resolution validation
Operational plumbing to land the parser fixes and to make silent resolution
failures visible.
- Reparse: SecFundamentalsImporter(reparse=True) restages every accession with
the current parser and rewrites the ones that now reconstruct differently,
writing the full column set so a row is never half old-parse. Snapshots stay
immutable with respect to SEC; the stored row is our reconstruction, and after
a parser fix keeping it is a stale cache, not history. run_import(force=True)
bypasses the unchanged-revision no-op, since the staleness is on our side, not
the source's. Exposed as scripts/reparse_fundamentals.py, dry-run by default.
- CIK overrides: sec_universe reads a {symbol: cik} pin from
SystemSetting['sec_cik_overrides'], applied ahead of company_tickers.json, for
when SEC maps a ticker to a successor shell with no filings (XOM -> a zero-
filing "ExxonMobil Holdings Corp" while every 10-K/Q is under CIK 34088).
- Resolution validation: a tracked issuer resolving to a registrant with no XBRL
filings now records no_xbrl_filings and raises a warning naming the CIKs and
the override setting, instead of silently yielding nothing on every run.
- _diff_fields compares datetime instants, not representations: accepted_at
round-trips naive from SQLite but tz-aware from Postgres, which otherwise made
a reparse of identical data report every row as changed (and false-positived
the pre-existing discrepancy warning).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -16,6 +16,7 @@ changes on the framework's failure commit). The proposals are applied only in
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Iterable
|
||||
@@ -23,11 +24,21 @@ from typing import Iterable
|
||||
from sqlalchemy import select, update
|
||||
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import settings_store
|
||||
from app.services.earnings_alignment import normalise_symbol
|
||||
from app.services.sec_client import SecClient
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# JSON {symbol: cik} pinning a ticker to a specific registrant, overriding
|
||||
# company_tickers.json. Needed when SEC maps a ticker to a successor entity that
|
||||
# has not filed: XOM points at CIK 2115436 "ExxonMobil Holdings Corp" (zero XBRL
|
||||
# filings) while every 10-K/10-Q — including one filed 2026-05-04 — is still under
|
||||
# CIK 34088. Which registrant is the real filer is a judgement about a corporate
|
||||
# event, so it is pinned explicitly rather than guessed. The importer's
|
||||
# `no_xbrl_filings` warning is what tells you a pin is needed.
|
||||
CIK_OVERRIDES_KEY = "sec_cik_overrides"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolvedUniverse:
|
||||
@@ -43,6 +54,7 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
|
||||
"""Resolve tracked tickers to CIKs via company_tickers.json. **Read-only** —
|
||||
returns the mapping + proposed `tickers.cik` writes; mutates nothing."""
|
||||
ticker_to_cik = await client.company_tickers()
|
||||
overrides = await cik_overrides(db)
|
||||
rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all()
|
||||
|
||||
result = ResolvedUniverse()
|
||||
@@ -50,7 +62,7 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
|
||||
if not symbol:
|
||||
continue
|
||||
sym = normalise_symbol(symbol)
|
||||
cik = ticker_to_cik.get(sym)
|
||||
cik = overrides.get(sym) or ticker_to_cik.get(sym)
|
||||
if cik is None:
|
||||
continue # ADRs / non-SEC issuers — snapshots simply absent
|
||||
result.symbol_to_cik[sym] = cik
|
||||
@@ -65,6 +77,34 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
|
||||
return result
|
||||
|
||||
|
||||
async def cik_overrides(db) -> dict[str, int]:
|
||||
"""Manual ``{symbol: cik}`` pins from ``SystemSetting[CIK_OVERRIDES_KEY]``.
|
||||
|
||||
A malformed setting must never take the importer down, so anything unparseable
|
||||
is logged and ignored — the run then falls back to company_tickers.json.
|
||||
"""
|
||||
raw = await settings_store.get_value(db, CIK_OVERRIDES_KEY)
|
||||
if not raw:
|
||||
return {}
|
||||
try:
|
||||
loaded = json.loads(raw)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("%s is not valid JSON — ignoring CIK overrides", CIK_OVERRIDES_KEY)
|
||||
return {}
|
||||
if not isinstance(loaded, dict):
|
||||
logger.warning("%s must be a {symbol: cik} object — ignoring", CIK_OVERRIDES_KEY)
|
||||
return {}
|
||||
out: dict[str, int] = {}
|
||||
for symbol, cik in loaded.items():
|
||||
try:
|
||||
out[normalise_symbol(str(symbol))] = int(cik)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("%s: bad entry %r -> %r — ignoring", CIK_OVERRIDES_KEY, symbol, cik)
|
||||
if out:
|
||||
logger.info("resolve_ciks: %d CIK override(s) applied: %s", len(out), sorted(out))
|
||||
return out
|
||||
|
||||
|
||||
async def fetch_sic_updates(
|
||||
client: SecClient, cik_to_ticker_ids: dict[int, Iterable[int]]
|
||||
) -> list[tuple[int, str | None, str | None]]:
|
||||
|
||||
Reference in New Issue
Block a user