feat(sec): reparse path, CIK overrides, resolution validation
Operational plumbing to land the parser fixes and to make silent resolution
failures visible.
- Reparse: SecFundamentalsImporter(reparse=True) restages every accession with
the current parser and rewrites the ones that now reconstruct differently,
writing the full column set so a row is never half old-parse. Snapshots stay
immutable with respect to SEC; the stored row is our reconstruction, and after
a parser fix keeping it is a stale cache, not history. run_import(force=True)
bypasses the unchanged-revision no-op, since the staleness is on our side, not
the source's. Exposed as scripts/reparse_fundamentals.py, dry-run by default.
- CIK overrides: sec_universe reads a {symbol: cik} pin from
SystemSetting['sec_cik_overrides'], applied ahead of company_tickers.json, for
when SEC maps a ticker to a successor shell with no filings (XOM -> a zero-
filing "ExxonMobil Holdings Corp" while every 10-K/Q is under CIK 34088).
- Resolution validation: a tracked issuer resolving to a registrant with no XBRL
filings now records no_xbrl_filings and raises a warning naming the CIKs and
the override setting, instead of silently yielding nothing on every run.
- _diff_fields compares datetime instants, not representations: accepted_at
round-trips naive from SQLite but tz-aware from Postgres, which otherwise made
a reparse of identical data report every row as changed (and false-positived
the pre-existing discrepancy warning).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -145,11 +145,17 @@ async def run_import(
|
|||||||
importer: SourceImporter,
|
importer: SourceImporter,
|
||||||
*,
|
*,
|
||||||
engine: AsyncEngine | None = None,
|
engine: AsyncEngine | None = None,
|
||||||
|
force: bool = False,
|
||||||
) -> DataImportRun | None:
|
) -> DataImportRun | None:
|
||||||
"""Run one import for ``importer``.
|
"""Run one import for ``importer``.
|
||||||
|
|
||||||
Returns the recorded ``DataImportRun`` (promoted / no_op / failed), or None
|
Returns the recorded ``DataImportRun`` (promoted / no_op / failed), or None
|
||||||
when the per-source advisory lock is already held (another run is active).
|
when the per-source advisory lock is already held (another run is active).
|
||||||
|
|
||||||
|
``force`` runs even when the revision is unchanged. The revision tracks the
|
||||||
|
*source*, so a re-import driven by a change on our side — a parser fix that
|
||||||
|
makes stored rows stale — is a no_op under the normal gate. Manually invoked
|
||||||
|
only; scheduled jobs must leave it False so an unchanged source stays a no_op.
|
||||||
"""
|
"""
|
||||||
engine = engine or app_engine
|
engine = engine or app_engine
|
||||||
source = importer.source
|
source = importer.source
|
||||||
@@ -189,7 +195,7 @@ async def run_import(
|
|||||||
revision = await importer.detect_revision(session)
|
revision = await importer.detect_revision(session)
|
||||||
run.revision = revision
|
run.revision = revision
|
||||||
last_rev = await _last_promoted_revision(session, source)
|
last_rev = await _last_promoted_revision(session, source)
|
||||||
if revision is not None and revision == last_rev:
|
if not force and revision is not None and revision == last_rev:
|
||||||
run.status = STATUS_NO_OP
|
run.status = STATUS_NO_OP
|
||||||
run.completed_at = _now()
|
run.completed_at = _now()
|
||||||
await session.commit()
|
await session.commit()
|
||||||
|
|||||||
@@ -21,6 +21,12 @@ Guardrails (design + reviews):
|
|||||||
- ``promote`` inserts snapshots ``ON CONFLICT (accession) DO NOTHING`` (immutable),
|
- ``promote`` inserts snapshots ``ON CONFLICT (accession) DO NOTHING`` (immutable),
|
||||||
reports differing existing accessions, and applies ticker updates in the same
|
reports differing existing accessions, and applies ticker updates in the same
|
||||||
transaction.
|
transaction.
|
||||||
|
- ``reparse=True`` is the one exception to immutability, and it is deliberate:
|
||||||
|
it restages every accession with the current parser and **rewrites** the rows
|
||||||
|
that now reconstruct differently. Immutability protects SEC's record (one row
|
||||||
|
per accession, amendments retained) — but the stored row is *our* reconstruction,
|
||||||
|
so after a parser fix, keeping it is preserving a stale cache, not history.
|
||||||
|
Manually invoked through ``scripts/reparse_fundamentals.py``; never scheduled.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -31,7 +37,7 @@ from dataclasses import dataclass, field
|
|||||||
from datetime import date, datetime, timedelta, timezone
|
from datetime import date, datetime, timedelta, timezone
|
||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
|
|
||||||
from sqlalchemy import select
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
from app.database import insert_for_session
|
from app.database import insert_for_session
|
||||||
from app.models.data_import_run import DataImportRun
|
from app.models.data_import_run import DataImportRun
|
||||||
@@ -57,7 +63,7 @@ _SNAPSHOT_COLS = (
|
|||||||
"period_end", "fiscal_year", "fiscal_period", "revenue", "net_income",
|
"period_end", "fiscal_year", "fiscal_period", "revenue", "net_income",
|
||||||
"operating_income", "diluted_eps", "cfo", "capex", "depreciation_amortization",
|
"operating_income", "diluted_eps", "cfo", "capex", "depreciation_amortization",
|
||||||
"cash_and_st_investments", "total_debt", "shares_outstanding",
|
"cash_and_st_investments", "total_debt", "shares_outstanding",
|
||||||
"shares_outstanding_date",
|
"shares_outstanding_date", "weighted_avg_diluted_shares",
|
||||||
)
|
)
|
||||||
# Compare ALL source fields (every column except the accession key) to flag a
|
# Compare ALL source fields (every column except the accession key) to flag a
|
||||||
# differing existing accession — immutable, so we report, never mutate.
|
# differing existing accession — immutable, so we report, never mutate.
|
||||||
@@ -75,6 +81,10 @@ class StagedFundamentals:
|
|||||||
missing_xbrl: list[dict[str, str]] = field(default_factory=list)
|
missing_xbrl: list[dict[str, str]] = field(default_factory=list)
|
||||||
invalid_payloads: list[dict[str, str]] = field(default_factory=list)
|
invalid_payloads: list[dict[str, str]] = field(default_factory=list)
|
||||||
existing_accessions: set[str] = field(default_factory=set)
|
existing_accessions: set[str] = field(default_factory=set)
|
||||||
|
# Tracked issuers whose registrant has NO XBRL 10-K/10-Q at all: they can
|
||||||
|
# never yield a snapshot, so this is a resolution problem (a ticker pointed
|
||||||
|
# at a successor shell), not missing data. See sec_universe.CIK_OVERRIDES_KEY.
|
||||||
|
no_xbrl_filings: list[dict[str, Any]] = field(default_factory=list)
|
||||||
discrepancies: list[dict[str, Any]] = field(default_factory=list)
|
discrepancies: list[dict[str, Any]] = field(default_factory=list)
|
||||||
backfill: bool = False
|
backfill: bool = False
|
||||||
issuers_fetched: int = 0
|
issuers_fetched: int = 0
|
||||||
@@ -93,9 +103,17 @@ class SecFundamentalsImporter:
|
|||||||
*,
|
*,
|
||||||
client_factory: Callable[[], SecClient] | None = None,
|
client_factory: Callable[[], SecClient] | None = None,
|
||||||
today: date | None = None,
|
today: date | None = None,
|
||||||
|
reparse: bool = False,
|
||||||
) -> None:
|
) -> None:
|
||||||
self._client_factory = client_factory or (lambda: SecClient())
|
self._client_factory = client_factory or (lambda: SecClient())
|
||||||
self.today = today or _now().date()
|
self.today = today or _now().date()
|
||||||
|
# Reparse: re-derive every stored accession with the CURRENT parser and
|
||||||
|
# rewrite the ones that now reconstruct differently. Snapshots are
|
||||||
|
# immutable with respect to SEC (one row per accession, amendments kept),
|
||||||
|
# but the stored row is *our reconstruction* — when a parser bug is fixed,
|
||||||
|
# leaving it stale is not immutability, it is a stale cache. Manually
|
||||||
|
# invoked via scripts/reparse_fundamentals.py; never scheduled.
|
||||||
|
self.reparse = reparse
|
||||||
# cached by detect_revision, consumed by stage:
|
# cached by detect_revision, consumed by stage:
|
||||||
self._resolved: ResolvedUniverse | None = None
|
self._resolved: ResolvedUniverse | None = None
|
||||||
self._index_rows: list[dict[str, Any]] = []
|
self._index_rows: list[dict[str, Any]] = []
|
||||||
@@ -111,7 +129,10 @@ class SecFundamentalsImporter:
|
|||||||
self._latest_index_date = await client.latest_index_date(self.today)
|
self._latest_index_date = await client.latest_index_date(self.today)
|
||||||
if self._latest_index_date is None:
|
if self._latest_index_date is None:
|
||||||
raise SecError("no EDGAR daily index available")
|
raise SecError("no EDGAR daily index available")
|
||||||
if last_processed is None:
|
# Reparse needs every accession restaged, not just those filed since
|
||||||
|
# the last run — the facts a fixed parser now accepts were never
|
||||||
|
# stored, so a reparse cannot be served from the database.
|
||||||
|
if last_processed is None or self.reparse:
|
||||||
self._backfill = True
|
self._backfill = True
|
||||||
self._index_rows = []
|
self._index_rows = []
|
||||||
else:
|
else:
|
||||||
@@ -175,6 +196,10 @@ class SecFundamentalsImporter:
|
|||||||
return
|
return
|
||||||
sub = await client.submissions(cik, include_history=is_backfill)
|
sub = await client.submissions(cik, include_history=is_backfill)
|
||||||
xbrl_meta, nonxbrl = _filing_meta(sub)
|
xbrl_meta, nonxbrl = _filing_meta(sub)
|
||||||
|
if not xbrl_meta:
|
||||||
|
staged.no_xbrl_filings.append(
|
||||||
|
{"cik": cik10(cik), "name": sub.get("name"), "tickers": sub.get("tickers")}
|
||||||
|
)
|
||||||
|
|
||||||
if is_backfill:
|
if is_backfill:
|
||||||
accns = set(xbrl_meta)
|
accns = set(xbrl_meta)
|
||||||
@@ -191,7 +216,11 @@ class SecFundamentalsImporter:
|
|||||||
# have lagged; fail+retry rather than record nothing for it.
|
# have lagged; fail+retry rather than record nothing for it.
|
||||||
staged.missing_xbrl.append({"cik": cik10(cik), "accession": accn})
|
staged.missing_xbrl.append({"cik": cik10(cik), "accession": accn})
|
||||||
|
|
||||||
result = parser.parse_snapshots(cf, xbrl_meta, accns)
|
# fiscalYearEnd (MMDD) is what lets the parser derive period identity from
|
||||||
|
# reportDate instead of SEC's unreliable fy/fp fields.
|
||||||
|
result = parser.parse_snapshots(
|
||||||
|
cf, xbrl_meta, accns, fiscal_year_end=sub.get("fiscal_year_end")
|
||||||
|
)
|
||||||
staged.rows.extend(result.rows)
|
staged.rows.extend(result.rows)
|
||||||
staged.skipped_filings.extend(result.skipped_filings)
|
staged.skipped_filings.extend(result.skipped_filings)
|
||||||
staged.field_issues.extend(result.field_issues)
|
staged.field_issues.extend(result.field_issues)
|
||||||
@@ -241,6 +270,8 @@ class SecFundamentalsImporter:
|
|||||||
"skipped_filings": len(staged.skipped_filings),
|
"skipped_filings": len(staged.skipped_filings),
|
||||||
"field_issues": len(staged.field_issues),
|
"field_issues": len(staged.field_issues),
|
||||||
"skipped_non_xbrl": len(staged.skipped_non_xbrl),
|
"skipped_non_xbrl": len(staged.skipped_non_xbrl),
|
||||||
|
"no_xbrl_filings": staged.no_xbrl_filings[:50],
|
||||||
|
"no_xbrl_filings_count": len(staged.no_xbrl_filings),
|
||||||
"missing_xbrl": len(staged.missing_xbrl),
|
"missing_xbrl": len(staged.missing_xbrl),
|
||||||
"invalid_payloads": staged.invalid_payloads,
|
"invalid_payloads": staged.invalid_payloads,
|
||||||
"cik_updates": len(staged.resolved.cik_updates),
|
"cik_updates": len(staged.resolved.cik_updates),
|
||||||
@@ -257,9 +288,26 @@ class SecFundamentalsImporter:
|
|||||||
|
|
||||||
async def promote(self, db, staged: StagedFundamentals, run_id: int) -> dict[str, int]:
|
async def promote(self, db, staged: StagedFundamentals, run_id: int) -> dict[str, int]:
|
||||||
inserted = 0
|
inserted = 0
|
||||||
|
updated = 0
|
||||||
|
# Only accessions whose reconstruction actually changed are rewritten;
|
||||||
|
# an unchanged stored row is left completely alone.
|
||||||
|
changed = {d["accession"] for d in staged.discrepancies} if self.reparse else set()
|
||||||
for row in staged.rows:
|
for row in staged.rows:
|
||||||
if row.accession in staged.existing_accessions:
|
if row.accession in staged.existing_accessions:
|
||||||
continue # immutable — keep the original row
|
if row.accession in changed:
|
||||||
|
# Write the FULL column set (_row_values covers _SNAPSHOT_COLS)
|
||||||
|
# so a rewritten row is never half old-parse, half new-parse.
|
||||||
|
# created_at stays at the original insert; import_run_id
|
||||||
|
# attributes the rewrite.
|
||||||
|
values = _row_values(row, run_id)
|
||||||
|
values.pop("created_at", None)
|
||||||
|
await db.execute(
|
||||||
|
update(FundamentalSnapshot)
|
||||||
|
.where(FundamentalSnapshot.accession == row.accession)
|
||||||
|
.values(**values)
|
||||||
|
)
|
||||||
|
updated += 1
|
||||||
|
continue # otherwise immutable — keep the original row
|
||||||
stmt = insert_for_session(db, FundamentalSnapshot).values(**_row_values(row, run_id))
|
stmt = insert_for_session(db, FundamentalSnapshot).values(**_row_values(row, run_id))
|
||||||
stmt = stmt.on_conflict_do_nothing(index_elements=["accession"]) # race belt-and-suspenders
|
stmt = stmt.on_conflict_do_nothing(index_elements=["accession"]) # race belt-and-suspenders
|
||||||
await db.execute(stmt)
|
await db.execute(stmt)
|
||||||
@@ -269,24 +317,48 @@ class SecFundamentalsImporter:
|
|||||||
# any existing accession reconstructed differently — kept immutable.
|
# any existing accession reconstructed differently — kept immutable.
|
||||||
if staged.discrepancies:
|
if staged.discrepancies:
|
||||||
accns = ", ".join(d["accession"] for d in staged.discrepancies[:10])
|
accns = ", ".join(d["accession"] for d in staged.discrepancies[:10])
|
||||||
|
disposition = (
|
||||||
|
f"REWRITTEN by reparse run {run_id}" if self.reparse else "kept immutable"
|
||||||
|
)
|
||||||
db.add(SystemEvent(
|
db.add(SystemEvent(
|
||||||
severity="warning",
|
severity="warning",
|
||||||
source="sec_facts",
|
source="sec_facts",
|
||||||
code="snapshot_discrepancy",
|
code="snapshot_reparse" if self.reparse else "snapshot_discrepancy",
|
||||||
message=(
|
message=(
|
||||||
f"{len(staged.discrepancies)} stored accession(s) reconstructed "
|
f"{len(staged.discrepancies)} stored accession(s) reconstructed "
|
||||||
f"differently; kept immutable: {accns}"
|
f"differently; {disposition}: {accns}"
|
||||||
)[:4000],
|
)[:4000],
|
||||||
dedup_key=f"sec_facts:discrepancy:{run_id}",
|
dedup_key=f"sec_facts:discrepancy:{run_id}",
|
||||||
created_at=_now(),
|
created_at=_now(),
|
||||||
))
|
))
|
||||||
|
|
||||||
|
# A tracked issuer whose registrant has no XBRL filings can never produce a
|
||||||
|
# snapshot, and it is restaged on every run forever. That is a resolution
|
||||||
|
# problem, not missing data, and it is silent without this.
|
||||||
|
if staged.no_xbrl_filings:
|
||||||
|
named = ", ".join(
|
||||||
|
f"{e['cik']} ({e.get('name') or '?'})" for e in staged.no_xbrl_filings[:10]
|
||||||
|
)
|
||||||
|
db.add(SystemEvent(
|
||||||
|
severity="warning",
|
||||||
|
source="sec_facts",
|
||||||
|
code="no_xbrl_filings",
|
||||||
|
message=(
|
||||||
|
f"{len(staged.no_xbrl_filings)} tracked issuer(s) resolved to a "
|
||||||
|
f"registrant with no XBRL 10-K/10-Q — pin the right CIK via the "
|
||||||
|
f"'{sec_universe.CIK_OVERRIDES_KEY}' setting: {named}"
|
||||||
|
)[:4000],
|
||||||
|
dedup_key=f"sec_facts:no_xbrl_filings:{run_id}",
|
||||||
|
created_at=_now(),
|
||||||
|
))
|
||||||
|
|
||||||
ticker_counts = await sec_universe.apply_ticker_updates(
|
ticker_counts = await sec_universe.apply_ticker_updates(
|
||||||
db, staged.resolved, staged.sic_updates
|
db, staged.resolved, staged.sic_updates
|
||||||
)
|
)
|
||||||
return {
|
return {
|
||||||
"inserted": inserted,
|
"inserted": inserted,
|
||||||
"existing_unchanged": len(staged.existing_accessions),
|
"updated": updated,
|
||||||
|
"existing_unchanged": len(staged.existing_accessions) - updated,
|
||||||
"discrepancies": len(staged.discrepancies),
|
"discrepancies": len(staged.discrepancies),
|
||||||
**ticker_counts,
|
**ticker_counts,
|
||||||
}
|
}
|
||||||
@@ -395,5 +467,26 @@ def _row_values(row: SnapshotRow, run_id: int) -> dict[str, Any]:
|
|||||||
|
|
||||||
|
|
||||||
def _diff_fields(row: SnapshotRow, old: FundamentalSnapshot) -> list[str]:
|
def _diff_fields(row: SnapshotRow, old: FundamentalSnapshot) -> list[str]:
|
||||||
"""Source fields where a re-parsed row differs from the stored (immutable) row."""
|
"""Source fields where a re-parsed row differs from the stored row."""
|
||||||
return [col for col in _COMPARE_COLS if getattr(row, col) != getattr(old, col)]
|
return [
|
||||||
|
col for col in _COMPARE_COLS
|
||||||
|
if not _same_value(getattr(row, col), getattr(old, col))
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _same_value(parsed: Any, stored: Any) -> bool:
|
||||||
|
"""Compare a freshly parsed value against its stored round-trip.
|
||||||
|
|
||||||
|
Datetimes need care: every timestamp here is UTC by construction, but
|
||||||
|
``DateTime(timezone=True)`` only preserves tzinfo on Postgres — SQLite hands
|
||||||
|
back a naive value. Comparing representations would report an unchanged row
|
||||||
|
as differing, which would both spam the discrepancy warning and make a
|
||||||
|
reparse rewrite every row it touched. Compare instants instead.
|
||||||
|
"""
|
||||||
|
if isinstance(parsed, datetime) and isinstance(stored, datetime):
|
||||||
|
return _as_utc(parsed) == _as_utc(stored)
|
||||||
|
return parsed == stored
|
||||||
|
|
||||||
|
|
||||||
|
def _as_utc(value: datetime) -> datetime:
|
||||||
|
return value if value.tzinfo is not None else value.replace(tzinfo=timezone.utc)
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ changes on the framework's failure commit). The proposals are applied only in
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
import json
|
||||||
import logging
|
import logging
|
||||||
from dataclasses import dataclass, field
|
from dataclasses import dataclass, field
|
||||||
from typing import Iterable
|
from typing import Iterable
|
||||||
@@ -23,11 +24,21 @@ from typing import Iterable
|
|||||||
from sqlalchemy import select, update
|
from sqlalchemy import select, update
|
||||||
|
|
||||||
from app.models.ticker import Ticker
|
from app.models.ticker import Ticker
|
||||||
|
from app.services import settings_store
|
||||||
from app.services.earnings_alignment import normalise_symbol
|
from app.services.earnings_alignment import normalise_symbol
|
||||||
from app.services.sec_client import SecClient
|
from app.services.sec_client import SecClient
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# JSON {symbol: cik} pinning a ticker to a specific registrant, overriding
|
||||||
|
# company_tickers.json. Needed when SEC maps a ticker to a successor entity that
|
||||||
|
# has not filed: XOM points at CIK 2115436 "ExxonMobil Holdings Corp" (zero XBRL
|
||||||
|
# filings) while every 10-K/10-Q — including one filed 2026-05-04 — is still under
|
||||||
|
# CIK 34088. Which registrant is the real filer is a judgement about a corporate
|
||||||
|
# event, so it is pinned explicitly rather than guessed. The importer's
|
||||||
|
# `no_xbrl_filings` warning is what tells you a pin is needed.
|
||||||
|
CIK_OVERRIDES_KEY = "sec_cik_overrides"
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
class ResolvedUniverse:
|
class ResolvedUniverse:
|
||||||
@@ -43,6 +54,7 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
|
|||||||
"""Resolve tracked tickers to CIKs via company_tickers.json. **Read-only** —
|
"""Resolve tracked tickers to CIKs via company_tickers.json. **Read-only** —
|
||||||
returns the mapping + proposed `tickers.cik` writes; mutates nothing."""
|
returns the mapping + proposed `tickers.cik` writes; mutates nothing."""
|
||||||
ticker_to_cik = await client.company_tickers()
|
ticker_to_cik = await client.company_tickers()
|
||||||
|
overrides = await cik_overrides(db)
|
||||||
rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all()
|
rows = (await db.execute(select(Ticker.id, Ticker.symbol, Ticker.cik))).all()
|
||||||
|
|
||||||
result = ResolvedUniverse()
|
result = ResolvedUniverse()
|
||||||
@@ -50,7 +62,7 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
|
|||||||
if not symbol:
|
if not symbol:
|
||||||
continue
|
continue
|
||||||
sym = normalise_symbol(symbol)
|
sym = normalise_symbol(symbol)
|
||||||
cik = ticker_to_cik.get(sym)
|
cik = overrides.get(sym) or ticker_to_cik.get(sym)
|
||||||
if cik is None:
|
if cik is None:
|
||||||
continue # ADRs / non-SEC issuers — snapshots simply absent
|
continue # ADRs / non-SEC issuers — snapshots simply absent
|
||||||
result.symbol_to_cik[sym] = cik
|
result.symbol_to_cik[sym] = cik
|
||||||
@@ -65,6 +77,34 @@ async def resolve_ciks(db, client: SecClient) -> ResolvedUniverse:
|
|||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
async def cik_overrides(db) -> dict[str, int]:
|
||||||
|
"""Manual ``{symbol: cik}`` pins from ``SystemSetting[CIK_OVERRIDES_KEY]``.
|
||||||
|
|
||||||
|
A malformed setting must never take the importer down, so anything unparseable
|
||||||
|
is logged and ignored — the run then falls back to company_tickers.json.
|
||||||
|
"""
|
||||||
|
raw = await settings_store.get_value(db, CIK_OVERRIDES_KEY)
|
||||||
|
if not raw:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
loaded = json.loads(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
logger.warning("%s is not valid JSON — ignoring CIK overrides", CIK_OVERRIDES_KEY)
|
||||||
|
return {}
|
||||||
|
if not isinstance(loaded, dict):
|
||||||
|
logger.warning("%s must be a {symbol: cik} object — ignoring", CIK_OVERRIDES_KEY)
|
||||||
|
return {}
|
||||||
|
out: dict[str, int] = {}
|
||||||
|
for symbol, cik in loaded.items():
|
||||||
|
try:
|
||||||
|
out[normalise_symbol(str(symbol))] = int(cik)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
logger.warning("%s: bad entry %r -> %r — ignoring", CIK_OVERRIDES_KEY, symbol, cik)
|
||||||
|
if out:
|
||||||
|
logger.info("resolve_ciks: %d CIK override(s) applied: %s", len(out), sorted(out))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
async def fetch_sic_updates(
|
async def fetch_sic_updates(
|
||||||
client: SecClient, cik_to_ticker_ids: dict[int, Iterable[int]]
|
client: SecClient, cik_to_ticker_ids: dict[int, Iterable[int]]
|
||||||
) -> list[tuple[int, str | None, str | None]]:
|
) -> list[tuple[int, str | None, str | None]]:
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
"""Re-derive every stored SEC snapshot with the current parser.
|
||||||
|
|
||||||
|
Snapshots are immutable per accession, so a parser fix does not reach rows that
|
||||||
|
are already stored: a normal import skips them and only logs a
|
||||||
|
``snapshot_discrepancy``. This script is the deliberate, manual exception --
|
||||||
|
it restages every accession from SEC Company Facts and rewrites the rows whose
|
||||||
|
reconstruction changed.
|
||||||
|
|
||||||
|
**Dry run by default.** Nothing is written unless ``--apply`` is passed. The dry
|
||||||
|
run stages and validates exactly as the real run does (both are read-only) and
|
||||||
|
reports the full blast radius: how many rows would change, which fields, and
|
||||||
|
per-symbol before/after samples.
|
||||||
|
|
||||||
|
Cost: a reparse cannot be served from the database -- the facts a fixed parser now
|
||||||
|
accepts were never stored -- so it refetches Company Facts for every tracked issuer
|
||||||
|
under the SEC fair-access throttle. Expect a long run and a lot of network.
|
||||||
|
|
||||||
|
Scope note: this rewrites ``fundamental_snapshots`` only. As of the A5 gate those
|
||||||
|
rows feed the fundamentals API/UI and the parity report; scoring still reads the
|
||||||
|
legacy ``fundamental_data`` table, so a reparse does not move composite scores or
|
||||||
|
backtests until the cutover happens.
|
||||||
|
|
||||||
|
Examples
|
||||||
|
--------
|
||||||
|
# dry run: report what would change, write nothing
|
||||||
|
python scripts/reparse_fundamentals.py
|
||||||
|
|
||||||
|
# dry run, showing more per-field detail
|
||||||
|
python scripts/reparse_fundamentals.py --samples 40
|
||||||
|
|
||||||
|
# actually rewrite the changed rows
|
||||||
|
python scripts/reparse_fundamentals.py --apply
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import asyncio
|
||||||
|
import sys
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||||
|
|
||||||
|
from app.database import async_session_factory # noqa: E402
|
||||||
|
from app.services.data_import import run_import # noqa: E402
|
||||||
|
from app.services.sec_fundamentals_importer import SecFundamentalsImporter # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_args() -> argparse.Namespace:
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
ap.add_argument("--apply", action="store_true",
|
||||||
|
help="rewrite changed rows (default: dry run, writes nothing)")
|
||||||
|
ap.add_argument("--samples", type=int, default=20,
|
||||||
|
help="how many changed accessions to show in detail (default 20)")
|
||||||
|
return ap.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
async def _dry_run(samples: int) -> int:
|
||||||
|
importer = SecFundamentalsImporter(reparse=True)
|
||||||
|
async with async_session_factory() as db:
|
||||||
|
print("staging every tracked issuer from SEC Company Facts (this is the slow part)...")
|
||||||
|
revision = await importer.detect_revision(db)
|
||||||
|
staged = await importer.stage(db)
|
||||||
|
result = await importer.validate(db, staged)
|
||||||
|
|
||||||
|
print(f"\nrevision : {revision}")
|
||||||
|
print(f"issuers fetched : {staged.issuers_fetched}")
|
||||||
|
print(f"rows reconstructed : {len(staged.rows)}")
|
||||||
|
print(f"already stored : {len(staged.existing_accessions)}")
|
||||||
|
print(f"WOULD BE REWRITTEN : {len(staged.discrepancies)}")
|
||||||
|
print(f"new inserts : {len(staged.rows) - len(staged.existing_accessions)}")
|
||||||
|
print(f"validation ok : {result.ok}")
|
||||||
|
if not result.ok:
|
||||||
|
print(f"validation messages : {result.messages}")
|
||||||
|
|
||||||
|
if staged.discrepancies:
|
||||||
|
field_counts = Counter(f for d in staged.discrepancies for f in d["fields"])
|
||||||
|
print("\nchanged fields (accession count per field):")
|
||||||
|
for name, count in field_counts.most_common():
|
||||||
|
print(f" {name:28s} {count}")
|
||||||
|
|
||||||
|
by_accession = {r.accession: r for r in staged.rows}
|
||||||
|
print(f"\nfirst {min(samples, len(staged.discrepancies))} changed accessions:")
|
||||||
|
for d in staged.discrepancies[:samples]:
|
||||||
|
row = by_accession.get(d["accession"])
|
||||||
|
where = f"{row.cik} {row.fiscal_year} {row.fiscal_period}" if row else "?"
|
||||||
|
print(f" {d['accession']} {where:28s} {', '.join(d['fields'])}")
|
||||||
|
|
||||||
|
print(
|
||||||
|
"\nDRY RUN -- nothing was written."
|
||||||
|
"\nCheck that the changes are the *kinds* you expect (recovered nulls,"
|
||||||
|
"\ncorrected values) and sample issuers you did not anticipate before"
|
||||||
|
"\nre-running with --apply."
|
||||||
|
)
|
||||||
|
return 0 if result.ok else 1
|
||||||
|
|
||||||
|
|
||||||
|
async def _apply() -> int:
|
||||||
|
# force=True: the revision tracks SEC, which has not changed — the staleness
|
||||||
|
# is on our side, so the normal no-op gate would skip this.
|
||||||
|
run = await run_import(SecFundamentalsImporter(reparse=True), force=True)
|
||||||
|
if run is None:
|
||||||
|
print("another sec_facts import holds the lock; nothing done")
|
||||||
|
return 1
|
||||||
|
print(f"run {run.id}: status={run.status}")
|
||||||
|
print(f" revision : {run.revision}")
|
||||||
|
print(f" row_counts : {run.row_counts_json}")
|
||||||
|
if run.error_details:
|
||||||
|
print(f" error : {run.error_details}")
|
||||||
|
return 0 if run.status == "promoted" else 1
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = _parse_args()
|
||||||
|
return asyncio.run(_apply() if args.apply else _dry_run(args.samples))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -411,3 +411,181 @@ async def test_discrepancy_in_shares_is_detected_and_reported(engine):
|
|||||||
assert k.shares_outstanding == 999.0 and k.import_run_id == 1 # immutable — not overwritten
|
assert k.shares_outstanding == 999.0 and k.import_run_id == 1 # immutable — not overwritten
|
||||||
events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all()
|
events = (await s.execute(select(SystemEvent).where(SystemEvent.code == "snapshot_discrepancy"))).scalars().all()
|
||||||
assert len(events) == 1 and events[0].severity == "warning"
|
assert len(events) == 1 and events[0].severity == "warning"
|
||||||
|
|
||||||
|
|
||||||
|
# --- reparse: rewriting rows a fixed parser reconstructs differently --------
|
||||||
|
|
||||||
|
# A 4-4-5 filer's YTD-Q3 span (36 weeks = 251 days). The old 20-day tolerance
|
||||||
|
# around 273 rejected it and stored revenue=None; 25 accepts it. Reparsing with
|
||||||
|
# the fixed parser is exactly the situation this mode exists for.
|
||||||
|
CF_Q3_445 = _rev("2025-09-01", "2026-05-10", 207431, 2026, "Q3", "Q3F")
|
||||||
|
SUB_445 = [_filing("Q3F", "10-Q", "2026-05-10", "2026-06-01", "2026-06-01T10:01:00.000Z")]
|
||||||
|
|
||||||
|
|
||||||
|
def _445_client():
|
||||||
|
return FakeSecClient(
|
||||||
|
tickers={"AAPL": 320193},
|
||||||
|
companyfacts={320193: _companyfacts([CF_Q3_445], [_shares("2026-05-15", 100, "Q3F", 2026, "Q3")])},
|
||||||
|
submissions={320193: _submissions(SUB_445)},
|
||||||
|
latest_index=date(2026, 6, 1),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _import_with_old_tolerance(engine, monkeypatch):
|
||||||
|
"""Seed the DB the way the pre-fix parser did: Q3 revenue rejected -> null."""
|
||||||
|
from app.services import sec_facts_parser
|
||||||
|
|
||||||
|
monkeypatch.setattr(sec_facts_parser, "_YTD_TOLERANCE_DAYS", 20)
|
||||||
|
run = await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
|
||||||
|
monkeypatch.undo()
|
||||||
|
return run
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reparse_rewrites_rows_the_fixed_parser_reads_differently(engine, monkeypatch):
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["AAPL"])
|
||||||
|
first = await _import_with_old_tolerance(engine, monkeypatch)
|
||||||
|
|
||||||
|
async with factory() as s:
|
||||||
|
stale = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
||||||
|
assert stale.revenue is None, "precondition: the old parser stored a null"
|
||||||
|
|
||||||
|
# Reparse with the current (fixed) parser. force=True because SEC has not
|
||||||
|
# changed -- the staleness is on our side, so the revision gate would no-op.
|
||||||
|
run = await run_import(
|
||||||
|
SecFundamentalsImporter(
|
||||||
|
client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
|
||||||
|
),
|
||||||
|
engine=engine,
|
||||||
|
force=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert run.status == STATUS_PROMOTED
|
||||||
|
assert '"updated": 1' in run.row_counts_json
|
||||||
|
async with factory() as s:
|
||||||
|
fixed = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
||||||
|
assert fixed.revenue == 207431 # rewritten in place
|
||||||
|
assert fixed.accession == stale.accession
|
||||||
|
assert fixed.import_run_id == run.id # rewrite is attributable
|
||||||
|
assert fixed.import_run_id != first.id
|
||||||
|
|
||||||
|
|
||||||
|
async def test_reparse_leaves_unchanged_rows_untouched(engine):
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["AAPL"])
|
||||||
|
first = await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
|
||||||
|
|
||||||
|
run = await run_import(
|
||||||
|
SecFundamentalsImporter(
|
||||||
|
client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
|
||||||
|
),
|
||||||
|
engine=engine,
|
||||||
|
force=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert '"updated": 0' in run.row_counts_json
|
||||||
|
async with factory() as s:
|
||||||
|
row = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
||||||
|
assert row.import_run_id == first.id # provenance preserved, no needless rewrite
|
||||||
|
|
||||||
|
|
||||||
|
async def test_without_reparse_a_differing_row_stays_immutable(engine, monkeypatch):
|
||||||
|
"""The default contract is unchanged: report the discrepancy, never mutate."""
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["AAPL"])
|
||||||
|
await _import_with_old_tolerance(engine, monkeypatch)
|
||||||
|
|
||||||
|
importer = SecFundamentalsImporter(
|
||||||
|
client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
|
||||||
|
)
|
||||||
|
async with _factory(engine)() as db:
|
||||||
|
await importer.detect_revision(db)
|
||||||
|
staged = await importer.stage(db)
|
||||||
|
importer.reparse = False # same staged diff, default disposition
|
||||||
|
counts = await importer.promote(db, staged, run_id=999)
|
||||||
|
await db.commit()
|
||||||
|
|
||||||
|
assert staged.discrepancies, "the diff should still be detected and reported"
|
||||||
|
assert counts["updated"] == 0
|
||||||
|
async with factory() as s:
|
||||||
|
row = (await s.execute(select(FundamentalSnapshot))).scalar_one()
|
||||||
|
assert row.revenue is None # untouched
|
||||||
|
|
||||||
|
|
||||||
|
async def test_force_bypasses_the_unchanged_revision_no_op(engine):
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["AAPL"])
|
||||||
|
await run_import(_importer(_445_client(), today=date(2026, 6, 2)), engine=engine)
|
||||||
|
|
||||||
|
same = _importer(_445_client(), today=date(2026, 6, 2))
|
||||||
|
assert (await run_import(same, engine=engine)).status == "no_op"
|
||||||
|
|
||||||
|
forced = SecFundamentalsImporter(
|
||||||
|
client_factory=lambda: _445_client(), today=date(2026, 6, 2), reparse=True
|
||||||
|
)
|
||||||
|
assert (await run_import(forced, engine=engine, force=True)).status == STATUS_PROMOTED
|
||||||
|
|
||||||
|
|
||||||
|
# --- CIK resolution: successor registrants with no filings -----------------
|
||||||
|
|
||||||
|
async def test_issuer_with_no_xbrl_filings_is_reported_not_silent(engine):
|
||||||
|
"""XOM resolved to CIK 2115436 'ExxonMobil Holdings Corp', which has zero
|
||||||
|
filings, so it produced no snapshots and nothing said why."""
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["AAPL"])
|
||||||
|
client = FakeSecClient(
|
||||||
|
tickers={"AAPL": 320193},
|
||||||
|
companyfacts={320193: _companyfacts([CF_K], [SH_K])},
|
||||||
|
submissions={320193: {**_submissions([]), "name": "Shell Holdings Corp"}},
|
||||||
|
latest_index=date(2026, 1, 31),
|
||||||
|
)
|
||||||
|
importer = _importer(client)
|
||||||
|
async with factory() as db:
|
||||||
|
await importer.detect_revision(db)
|
||||||
|
staged = await importer.stage(db)
|
||||||
|
result = await importer.validate(db, staged)
|
||||||
|
|
||||||
|
assert result.summary["no_xbrl_filings_count"] == 1
|
||||||
|
assert staged.no_xbrl_filings[0]["cik"] == "0000320193"
|
||||||
|
assert staged.no_xbrl_filings[0]["name"] == "Shell Holdings Corp"
|
||||||
|
|
||||||
|
|
||||||
|
async def test_cik_override_pins_a_ticker_to_the_real_filer(engine):
|
||||||
|
from app.models.settings import SystemSetting
|
||||||
|
from app.services.sec_universe import CIK_OVERRIDES_KEY, resolve_ciks
|
||||||
|
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["AAPL"])
|
||||||
|
async with factory() as s:
|
||||||
|
s.add(SystemSetting(key=CIK_OVERRIDES_KEY, value='{"AAPL": 34088}'))
|
||||||
|
await s.commit()
|
||||||
|
|
||||||
|
client = FakeSecClient(
|
||||||
|
tickers={"AAPL": 320193}, # SEC points at the wrong registrant
|
||||||
|
companyfacts={}, submissions={}, latest_index=date(2026, 1, 31),
|
||||||
|
)
|
||||||
|
async with factory() as db:
|
||||||
|
resolved = await resolve_ciks(db, client)
|
||||||
|
|
||||||
|
assert resolved.symbol_to_cik["AAPL"] == 34088
|
||||||
|
assert resolved.cik_updates == [(1, "0000034088")]
|
||||||
|
|
||||||
|
|
||||||
|
async def test_malformed_cik_override_is_ignored_not_fatal(engine):
|
||||||
|
from app.models.settings import SystemSetting
|
||||||
|
from app.services.sec_universe import CIK_OVERRIDES_KEY, resolve_ciks
|
||||||
|
|
||||||
|
factory = _factory(engine)
|
||||||
|
await _seed(factory, ["AAPL"])
|
||||||
|
async with factory() as s:
|
||||||
|
s.add(SystemSetting(key=CIK_OVERRIDES_KEY, value="not json at all"))
|
||||||
|
await s.commit()
|
||||||
|
|
||||||
|
client = FakeSecClient(
|
||||||
|
tickers={"AAPL": 320193}, companyfacts={}, submissions={},
|
||||||
|
latest_index=date(2026, 1, 31),
|
||||||
|
)
|
||||||
|
async with factory() as db:
|
||||||
|
resolved = await resolve_ciks(db, client)
|
||||||
|
|
||||||
|
assert resolved.symbol_to_cik["AAPL"] == 320193 # fell back to company_tickers
|
||||||
|
|||||||
Reference in New Issue
Block a user