dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: gate setups on SEC filing completeness

Browse Source
This commit is contained in:
dennisthiessen
2026-08-03 23:47:07 +02:00
parent 7d703ea524
commit 3a6900d45a
11 changed files with 723 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/sec_fundamentals_importer.py
+233 -55
View File
@@ -31,10 +31,9 @@ Guardrails (design + reviews):
stored as the parent's. Confirmed 2026-07-27 (NEE via FPL, DOW via Dow Chemical)
and it is not transient: an NEE filing misattributed in 2014 is still misfiled.
- **Bounded blocking.** Anything still unresolvable after ``MISSING_XBRL_RETRY_DAYS``
stops failing the run and is promoted around, with a named ``unresolved_filing``
warning. One filing SEC misfiled must not wedge every later import; the index
only moves forward, so a tolerated accession returns only via a reparse, and
only once SEC has re-filed it under the filer's own CIK.
stops failing the whole import and enters a durable retry queue. The scheduled
importer retries queued accessions automatically, while the affected issuer is
excluded from actionable setups until its filing is recovered.
- ``promote`` inserts snapshots ``ON CONFLICT (accession) DO NOTHING`` (immutable),
reports differing existing accessions, and applies ticker updates in the same
transaction.
@@ -48,18 +47,21 @@ Guardrails (design + reviews):
from __future__ import annotations
import json
import logging
from collections import Counter, defaultdict
from dataclasses import dataclass, field, replace
from datetime import date, datetime, timedelta, timezone
from typing import Any, Callable
from sqlalchemy import select, update
from sqlalchemy import delete, select, update
from app.database import insert_for_session
from app.models.data_import_run import DataImportRun
from app.models.fundamental_snapshot import FundamentalSnapshot
from app.models.sec_filing_gap import SecFilingGap
from app.models.system_event import SystemEvent
from app.services import fundamentals_quality_service
from app.services import sec_facts_parser as parser
from app.services import sec_universe
from app.services.data_import import STATUS_PROMOTED, ValidationResult
@@ -151,6 +153,8 @@ class SecFundamentalsImporter:
# accession -> the OTHER CIKs the daily index lists it under (co-registrants
# of a combined filing). Only populated for accessions a tracked issuer filed.
self._coregistrants: dict[str, list[int]] = {}
self._retry_rows: list[dict[str, Any]] = []
self._retry_queue_bootstrap_pending = False
self._latest_index_date: date | None = None
self._backfill = False
@@ -176,9 +180,23 @@ class SecFundamentalsImporter:
client, last_processed, self._latest_index_date
)
content = sec_universe.index_content_hash(self._index_rows)
return sec_universe.compose_revision(
revision = sec_universe.compose_revision(
self._latest_index_date, content, self._resolved.symbol_to_cik
)
self._retry_rows = []
if not self._backfill:
self._retry_queue_bootstrap_pending = not (
await fundamentals_quality_service.retry_queue_initialized(db)
)
self._retry_rows = await self._retry_backlog(
db,
set(self._resolved.cik_to_ticker_ids),
include_history=self._retry_queue_bootstrap_pending,
)
# Company Facts can change while the daily index revision stays fixed.
# Returning None deliberately bypasses the framework's no-op gate so a
# scheduled run retries every active gap.
return None if self._retry_rows or self._retry_queue_bootstrap_pending else revision
async def stage(self, db) -> StagedFundamentals:
assert self._resolved is not None, "detect_revision must run first"
@@ -193,6 +211,27 @@ class SecFundamentalsImporter:
if r["cik"] in cik_to_tids:
filed_by_cik[r["cik"]].append(r)
# Promoted-around filings live in a small durable retry queue. Historical
# validation payloads bootstrap gaps created before the queue existed.
# Merge them into the normal incremental work so the scheduled import,
# not an operator-run full reparse, heals them when SEC catches up.
if not self._backfill:
seen = {
(int(cik), row["accession"])
for cik, rows in filed_by_cik.items()
for row in rows
}
for row in self._retry_rows:
cik = int(row["cik"])
key = (cik, row["accession"])
if key in seen:
continue
filed_by_cik[cik].append(row)
seen.add(key)
coregistrants = [int(value) for value in row.get("coregistrants") or []]
if coregistrants:
self._coregistrants[row["accession"]] = coregistrants
existing = await self._ciks_with_snapshots(db, set(cik_to_tids))
if self._backfill:
backfill_ciks = set(cik_to_tids)
@@ -262,7 +301,13 @@ class SecFundamentalsImporter:
# or a co-registrant filing). NOT a Company-Facts lag — separate
# cause, separate fix, so it gets its own reason.
staged.missing_xbrl.append(
_missing(cik, index_row, "not_in_submissions", self.today)
_missing(
cik,
index_row,
"not_in_submissions",
self.today,
self._coregistrants.get(accn),
)
)
elif accn in present:
accns.add(accn)
@@ -289,6 +334,7 @@ class SecFundamentalsImporter:
"coregistrant_facts_rejected" if source_cik
else "not_in_companyfacts",
self.today,
self._coregistrants.get(accn),
))
# fiscalYearEnd (MMDD) is what lets the parser derive period identity from
@@ -396,12 +442,12 @@ class SecFundamentalsImporter:
"skipped_filings": len(staged.skipped_filings),
"field_issues": len(staged.field_issues),
"skipped_non_xbrl": len(staged.skipped_non_xbrl),
"no_xbrl_filings": staged.no_xbrl_filings[:50],
"no_xbrl_filings": staged.no_xbrl_filings,
"no_xbrl_filings_count": len(staged.no_xbrl_filings),
"missing_xbrl": staged.missing_xbrl[:50],
"missing_xbrl": staged.missing_xbrl,
"missing_xbrl_count": len(staged.missing_xbrl),
"missing_xbrl_blocking": len(blocking),
"recovered_from_coregistrant": staged.recovered[:50],
"recovered_from_coregistrant": staged.recovered,
"recovered_count": len(staged.recovered),
"invalid_payloads": staged.invalid_payloads,
"cik_updates": len(staged.resolved.cik_updates),
@@ -426,8 +472,8 @@ class SecFundamentalsImporter:
deferred_alert_messages=(
[
f"{len(aged_out)} tracked SEC filing(s) remain unresolved past "
f"the {MISSING_XBRL_RETRY_DAYS}-day retry window and risk being "
f"promoted around without automatic retry: "
f"the {MISSING_XBRL_RETRY_DAYS}-day retry window. They will "
f"enter automatic retry and block affected symbols from setups: "
f"{_missing_detail(aged_out)}"
]
if aged_out
@@ -462,6 +508,53 @@ class SecFundamentalsImporter:
await db.execute(stmt)
inserted += 1
# Synchronize the active retry queue in the same transaction as snapshot
# promotion. Reconstructed accessions leave the queue; aged-out gaps enter
# or refresh it and will be retried by the next scheduled import.
existing_gap_accessions = set(
(await db.execute(select(SecFilingGap.accession))).scalars().all()
)
resolved_accessions = {row.accession for row in staged.rows}
queue_resolved = 0
if resolved_accessions:
result = await db.execute(
delete(SecFilingGap).where(
SecFilingGap.accession.in_(resolved_accessions)
)
)
queue_resolved = int(result.rowcount or 0)
tolerated = _past_retry_window(staged.missing_xbrl)
now = _now()
for gap in tolerated:
stmt = insert_for_session(db, SecFilingGap).values(
cik=gap["cik"],
accession=gap["accession"],
form=gap.get("form"),
index_date=gap.get("index_date"),
reason=gap["reason"],
coregistrant_ciks_json=json.dumps(gap.get("coregistrants") or []),
first_seen_at=now,
last_attempted_at=now,
)
await db.execute(
stmt.on_conflict_do_update(
index_elements=["accession"],
set_={
"cik": stmt.excluded.cik,
"form": stmt.excluded.form,
"index_date": stmt.excluded.index_date,
"reason": stmt.excluded.reason,
"coregistrant_ciks_json": stmt.excluded.coregistrant_ciks_json,
"last_attempted_at": stmt.excluded.last_attempted_at,
},
)
)
newly_queued = [
gap for gap in tolerated
if gap["accession"] not in existing_gap_accessions
]
# Warn (in-transaction, so it commits atomically with the promotion) when
# any existing accession reconstructed differently — kept immutable.
if staged.discrepancies:
@@ -487,63 +580,47 @@ class SecFundamentalsImporter:
named = ", ".join(
f"{r['accession']} <- CIK {r['source_cik']}" for r in staged.recovered[:10]
)
db.add(SystemEvent(
severity="warning",
source="sec_facts",
code="coregistrant_recovery",
message=(
f"{len(staged.recovered)} filing(s) were absent from the filer's "
f"own Company Facts and were parsed from a co-registrant's file "
f"instead (share count checked against the issuer's history): {named}"
)[:4000],
dedup_key=f"sec_facts:coregistrant_recovery:{run_id}",
created_at=_now(),
))
logger.info(
"sec_facts: recovered %d filing(s) from co-registrants: %s",
len(staged.recovered),
named,
)
# Filings past the retry window: promoted WITHOUT them so one misfiled
# filing cannot wedge every later import. This is the deliberate trade —
# loud and named, because the index only moves forward and nothing will
# revisit them on its own.
tolerated = _past_retry_window(staged.missing_xbrl)
if tolerated:
# One warning when a gap first enters automatic retry. Repeating it every
# day adds noise; the queue remains the durable actionable state.
if newly_queued:
symbols_by_cik: dict[str, list[str]] = defaultdict(list)
for symbol, cik in staged.resolved.symbol_to_cik.items():
symbols_by_cik[cik10(cik)].append(symbol)
named = ", ".join(
f"{'/'.join(symbols_by_cik.get(gap['cik'], [])) or gap['cik']}"
f"/{gap['accession']}"
for gap in newly_queued[:10]
)
db.add(SystemEvent(
severity="warning",
source="sec_facts",
code="unresolved_filing",
message=(
f"{len(tolerated)} tracked filing(s) still unresolvable after "
f"{MISSING_XBRL_RETRY_DAYS} days; promoting without them rather "
f"than blocking every later import. They are NOT retried. "
f"scripts/reparse_fundamentals.py recovers them ONLY once SEC "
f"re-files under the filer's own CIK — it walks no index, so it "
f"cannot reach facts still sitting under a co-registrant: "
f"{_missing_detail(tolerated)}"
f"{len(newly_queued)} filing(s) entered automatic SEC retry. "
f"Affected symbols are blocked from new actionable setups until "
f"their filing is recovered: {named}"
)[:4000],
dedup_key=f"sec_facts:unresolved_filing:{run_id}",
created_at=_now(),
))
# A tracked issuer whose registrant has no XBRL filings can never produce a
# snapshot, and it is restaged on every run forever. That is a resolution
# problem, not missing data, and it is silent without this.
# A new registrant may have no XBRL filing yet. Keep it out of actionable
# setups, but log it instead of raising a recurring operator warning.
if staged.no_xbrl_filings:
named = ", ".join(
f"{e['cik']} ({e.get('name') or '?'})" for e in staged.no_xbrl_filings[:10]
)
db.add(SystemEvent(
severity="warning",
source="sec_facts",
code="no_xbrl_filings",
message=(
f"{len(staged.no_xbrl_filings)} tracked issuer(s) resolved to a "
f"registrant with no XBRL 10-K/10-Q. Either a successor shell "
f"(pin the real filer via the '{sec_universe.CIK_OVERRIDES_KEY}' "
f"setting) or a new registrant that has not filed its first "
f"10-K/10-Q yet, which needs nothing and clears itself: {named}"
)[:4000],
dedup_key=f"sec_facts:no_xbrl_filings:{run_id}",
created_at=_now(),
))
logger.info(
"sec_facts: %d registrant(s) have no XBRL history yet: %s",
len(staged.no_xbrl_filings),
named,
)
ticker_counts = await sec_universe.apply_ticker_updates(
db, staged.resolved, staged.sic_updates
@@ -553,11 +630,105 @@ class SecFundamentalsImporter:
"updated": updated,
"existing_unchanged": len(staged.existing_accessions) - updated,
"discrepancies": len(staged.discrepancies),
"retry_queue_added": len(newly_queued),
"retry_queue_resolved": queue_resolved,
**ticker_counts,
}
# -- helpers -----------------------------------------------------------
async def _retry_backlog(
self,
db,
tracked_ciks: set[int],
*,
include_history: bool,
) -> list[dict[str, Any]]:
"""Active gaps plus pre-queue gaps from promoted validation history."""
if not tracked_ciks:
return []
tracked = {cik10(cik) for cik in tracked_ciks}
candidates: dict[str, dict[str, Any]] = {}
queued = (
await db.execute(
select(SecFilingGap).where(SecFilingGap.cik.in_(tracked))
)
).scalars().all()
for gap in queued:
try:
coregistrants = json.loads(gap.coregistrant_ciks_json or "[]")
except (TypeError, ValueError):
coregistrants = []
candidates[gap.accession] = {
"cik": gap.cik,
"accession": gap.accession,
"form": gap.form,
"index_date": gap.index_date,
"reason": gap.reason,
"coregistrants": coregistrants,
}
# Bootstrap warnings produced before sec_filing_gaps existed. Promoted
# runs contain only aged-out gaps; deferred/failed runs are naturally
# retried because source_max_date has not advanced.
if include_history:
histories = (
await db.execute(
select(DataImportRun.validation_json)
.where(
DataImportRun.source == SOURCE,
DataImportRun.status == STATUS_PROMOTED,
DataImportRun.validation_json.is_not(None),
)
.order_by(DataImportRun.id.asc())
)
).scalars().all()
for payload in histories:
try:
summary = json.loads(payload)
except (TypeError, ValueError):
continue
for item in summary.get("missing_xbrl") or []:
accession = item.get("accession")
cik = str(item.get("cik") or "")
if not accession or cik not in tracked or accession in candidates:
continue
raw_date = item.get("index_date")
try:
index_date = (
date.fromisoformat(raw_date)
if isinstance(raw_date, str)
else raw_date
)
except ValueError:
index_date = None
candidates[accession] = {
"cik": cik,
"accession": accession,
"form": item.get("form"),
"index_date": index_date,
"reason": item.get("reason") or "not_in_companyfacts",
"coregistrants": item.get("coregistrants") or [],
}
if not candidates:
return []
resolved = set(
(
await db.execute(
select(FundamentalSnapshot.accession).where(
FundamentalSnapshot.accession.in_(list(candidates))
)
)
).scalars().all()
)
return [
item
for accession, item in candidates.items()
if accession not in resolved
]
async def _last_processed_index_date(self, db) -> date | None:
return (
await db.execute(
@@ -666,7 +837,13 @@ def _companyfacts_structure_error(cf: Any) -> str | None:
return None
def _missing(cik: int, row: dict[str, Any], reason: str, today: date) -> dict[str, Any]:
def _missing(
cik: int,
row: dict[str, Any],
reason: str,
today: date,
coregistrants: list[int] | None = None,
) -> dict[str, Any]:
"""One unresolvable index row, carrying everything needed to look the filing
up by hand (EDGAR accession + the index date it was seen on) and to decide
whether it is still young enough to be worth blocking on."""
@@ -679,6 +856,7 @@ def _missing(cik: int, row: dict[str, Any], reason: str, today: date) -> dict[st
# No index date (older cached rows) => age 0 => blocks, the safe default.
"age_days": (today - index_date).days if isinstance(index_date, date) else 0,
"reason": reason,
"coregistrants": list(coregistrants or []),
}