dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make SEC quality gating terminal-safe

Browse Source
This commit is contained in:
dennisthiessen
2026-08-04 07:58:18 +02:00
parent d1caac86b5
commit 4c0c0579f5
14 changed files with 629 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/sec_fundamentals_importer.py
+98 -73
View File
@@ -81,6 +81,7 @@ MIN_BACKFILL_COVERAGE = 0.5
# three); past that it is misfiled, not late, and blocking forever costs more
# than the missing filing does — see the unresolved-filing guardrail below.
MISSING_XBRL_RETRY_DAYS = 3
FILING_GAP_ESCALATE_DAYS = 14
# Share-count band a co-registrant-recovered row must land in, relative to the
# issuer's own last snapshot. Wide enough for buybacks/issuance, nowhere near
# wide enough to let a subsidiary shell's token float through (see _shares_continuous).
@@ -154,7 +155,6 @@ class SecFundamentalsImporter:
# of a combined filing). Only populated for accessions a tracked issuer filed.
self._coregistrants: dict[str, list[int]] = {}
self._retry_rows: list[dict[str, Any]] = []
self._retry_queue_bootstrap_pending = False
self._latest_index_date: date | None = None
self._backfill = False
@@ -185,18 +185,14 @@ class SecFundamentalsImporter:
)
self._retry_rows = []
if not self._backfill:
self._retry_queue_bootstrap_pending = not (
await fundamentals_quality_service.retry_queue_initialized(db)
)
self._retry_rows = await self._retry_backlog(
db,
set(self._resolved.cik_to_ticker_ids),
include_history=self._retry_queue_bootstrap_pending,
)
# Company Facts can change while the daily index revision stays fixed.
# Returning None deliberately bypasses the framework's no-op gate so a
# scheduled run retries every active gap.
return None if self._retry_rows or self._retry_queue_bootstrap_pending else revision
return None if self._retry_rows else revision
async def stage(self, db) -> StagedFundamentals:
assert self._resolved is not None, "detect_revision must run first"
@@ -211,10 +207,9 @@ class SecFundamentalsImporter:
if r["cik"] in cik_to_tids:
filed_by_cik[r["cik"]].append(r)
# Promoted-around filings live in a small durable retry queue. Historical
# validation payloads bootstrap gaps created before the queue existed.
# Merge them into the normal incremental work so the scheduled import,
# not an operator-run full reparse, heals them when SEC catches up.
# Promoted-around filings live in a small durable retry queue, including
# the one-time migration backfill. Merge them into normal incremental
# work so the scheduled importer heals them without operator action.
if not self._backfill:
seen = {
(int(cik), row["accession"])
@@ -286,6 +281,9 @@ class SecFundamentalsImporter:
fiscal_year_end = sub.get("fiscal_year_end")
recovered_rows: list[SnapshotRow] = []
index_rows = {
row["accession"]: row for row in filed_by_cik.get(cik, [])
}
if is_backfill:
accns = set(xbrl_meta)
else:
@@ -340,6 +338,19 @@ class SecFundamentalsImporter:
# fiscalYearEnd (MMDD) is what lets the parser derive period identity from
# reportDate instead of SEC's unreliable fy/fp fields.
result = parser.parse_snapshots(cf, xbrl_meta, accns, fiscal_year_end=fiscal_year_end)
for skipped in result.skipped_filings:
index_row = index_rows.get(skipped["accession"])
if index_row is not None:
# Facts are present but our parser cannot construct a snapshot.
# This is terminal for this run: promote around it immediately,
# keep the issuer blocked, and retry/escalate through the queue.
staged.missing_xbrl.append(_missing(
cik,
{**index_row, "_retry_queue": True},
"parser_unusable",
self.today,
self._coregistrants.get(skipped["accession"]),
))
staged.rows.extend(result.rows)
staged.rows.extend(recovered_rows)
staged.skipped_filings.extend(result.skipped_filings)
@@ -442,13 +453,19 @@ class SecFundamentalsImporter:
"skipped_filings": len(staged.skipped_filings),
"field_issues": len(staged.field_issues),
"skipped_non_xbrl": len(staged.skipped_non_xbrl),
"no_xbrl_filings": staged.no_xbrl_filings,
"no_xbrl_filings": staged.no_xbrl_filings[:50],
"no_xbrl_filings_count": len(staged.no_xbrl_filings),
"missing_xbrl": staged.missing_xbrl,
"missing_xbrl": staged.missing_xbrl[:50],
"missing_xbrl_count": len(staged.missing_xbrl),
"missing_xbrl_blocking": len(blocking),
"recovered_from_coregistrant": staged.recovered,
"recovered_from_coregistrant": staged.recovered[:50],
"recovered_count": len(staged.recovered),
# Complete compact gate input; detailed audit lists above stay capped.
"setup_blocked_ciks": sorted({
str(item["cik"])
for item in [*staged.missing_xbrl, *staged.no_xbrl_filings]
if item.get("cik")
}),
"invalid_payloads": staged.invalid_payloads,
"cik_updates": len(staged.resolved.cik_updates),
# differing existing accessions (immutable — kept, reported here)
@@ -508,13 +525,15 @@ class SecFundamentalsImporter:
await db.execute(stmt)
inserted += 1
# Synchronize the active retry queue in the same transaction as snapshot
# promotion. Reconstructed accessions leave the queue; aged-out gaps enter
# or refresh it and will be retried by the next scheduled import.
existing_gap_accessions = set(
(await db.execute(select(SecFilingGap.accession))).scalars().all()
)
# Synchronize the retry queue in the snapshot-promotion transaction.
existing_gaps = (await db.execute(select(SecFilingGap))).scalars().all()
existing_gap_accessions = {gap.accession for gap in existing_gaps}
resolved_accessions = {row.accession for row in staged.rows}
# A filing now classified non-XBRL can never yield a snapshot and is no
# longer a fundamentals completeness gap.
resolved_accessions.update(
item["accession"] for item in staged.skipped_non_xbrl
)
queue_resolved = 0
if resolved_accessions:
result = await db.execute(
@@ -524,8 +543,8 @@ class SecFundamentalsImporter:
)
queue_resolved = int(result.rowcount or 0)
tolerated = _past_retry_window(staged.missing_xbrl)
now = _now()
tolerated = _past_retry_window(staged.missing_xbrl)
for gap in tolerated:
stmt = insert_for_session(db, SecFilingGap).values(
cik=gap["cik"],
@@ -550,6 +569,20 @@ class SecFundamentalsImporter:
},
)
)
# Remove gaps made irrelevant by a later valid 10-K/10-Q. Quality reads
# already ignore them; physical cleanup keeps the queue small.
active_ids = {gap.id for gap in await fundamentals_quality_service.active_gaps(db)}
obsolete_ids = {
gap.id for gap in existing_gaps
if gap.id not in active_ids and gap.accession not in resolved_accessions
}
if obsolete_ids:
result = await db.execute(
delete(SecFilingGap).where(SecFilingGap.id.in_(obsolete_ids))
)
queue_resolved += int(result.rowcount or 0)
newly_queued = [
gap for gap in tolerated
if gap["accession"] not in existing_gap_accessions
@@ -574,6 +607,40 @@ class SecFundamentalsImporter:
created_at=_now(),
))
# Persistent current gaps get one actionable escalation rather than a
# daily warning. The nullable marker makes this durable and noise-free.
escalation_cutoff = now - timedelta(days=FILING_GAP_ESCALATE_DAYS)
aged_gaps = (
await db.execute(
select(SecFilingGap).where(
SecFilingGap.first_seen_at <= escalation_cutoff,
SecFilingGap.escalated_at.is_(None),
)
)
).scalars().all()
if aged_gaps:
named = ", ".join(
f"{gap.cik}/{gap.accession} ({gap.reason})"
for gap in aged_gaps[:10]
)
db.add(SystemEvent(
severity="warning",
source="sec_facts",
code="filing_gap_aged",
message=(
f"{len(aged_gaps)} SEC filing gap(s) remain unresolved after "
f"{FILING_GAP_ESCALATE_DAYS} days; affected setups remain paused. "
f"Review the filing/CIK mapping or parser: {named}"
)[:4000],
dedup_key=f"sec_facts:filing_gap_aged:{run_id}",
created_at=now,
))
await db.execute(
update(SecFilingGap)
.where(SecFilingGap.id.in_([gap.id for gap in aged_gaps]))
.values(escalated_at=now)
)
# Recovered rows are real data from an unexpected place — record where they
# came from, so a wrong recovery is auditable rather than invisible.
if staged.recovered:
@@ -641,20 +708,14 @@ class SecFundamentalsImporter:
self,
db,
tracked_ciks: set[int],
*,
include_history: bool,
) -> list[dict[str, Any]]:
"""Active gaps plus pre-queue gaps from promoted validation history."""
"""Active typed gaps; migration 028 owns historical bootstrap."""
if not tracked_ciks:
return []
tracked = {cik10(cik) for cik in tracked_ciks}
candidates: dict[str, dict[str, Any]] = {}
queued = (
await db.execute(
select(SecFilingGap).where(SecFilingGap.cik.in_(tracked))
)
).scalars().all()
queued = await fundamentals_quality_service.active_gaps(db, tracked)
for gap in queued:
try:
coregistrants = json.loads(gap.coregistrant_ciks_json or "[]")
@@ -667,51 +728,9 @@ class SecFundamentalsImporter:
"index_date": gap.index_date,
"reason": gap.reason,
"coregistrants": coregistrants,
"_retry_queue": True,
}
# Bootstrap warnings produced before sec_filing_gaps existed. Promoted
# runs contain only aged-out gaps; deferred/failed runs are naturally
# retried because source_max_date has not advanced.
if include_history:
histories = (
await db.execute(
select(DataImportRun.validation_json)
.where(
DataImportRun.source == SOURCE,
DataImportRun.status == STATUS_PROMOTED,
DataImportRun.validation_json.is_not(None),
)
.order_by(DataImportRun.id.asc())
)
).scalars().all()
for payload in histories:
try:
summary = json.loads(payload)
except (TypeError, ValueError):
continue
for item in summary.get("missing_xbrl") or []:
accession = item.get("accession")
cik = str(item.get("cik") or "")
if not accession or cik not in tracked or accession in candidates:
continue
raw_date = item.get("index_date")
try:
index_date = (
date.fromisoformat(raw_date)
if isinstance(raw_date, str)
else raw_date
)
except ValueError:
index_date = None
candidates[accession] = {
"cik": cik,
"accession": accession,
"form": item.get("form"),
"index_date": index_date,
"reason": item.get("reason") or "not_in_companyfacts",
"coregistrants": item.get("coregistrants") or [],
}
if not candidates:
return []
resolved = set(
@@ -848,13 +867,19 @@ def _missing(
up by hand (EDGAR accession + the index date it was seen on) and to decide
whether it is still young enough to be worth blocking on."""
index_date = row.get("index_date")
age_days = (
(today - index_date).days if isinstance(index_date, date) else 0
)
if row.get("_retry_queue"):
age_days = max(age_days, MISSING_XBRL_RETRY_DAYS + 1)
return {
"cik": cik10(cik),
"accession": row["accession"],
"form": row.get("form"),
"index_date": index_date,
# No index date (older cached rows) => age 0 => blocks, the safe default.
"age_days": (today - index_date).days if isinstance(index_date, date) else 0,
# A newly observed row without a date blocks safely. A durable queue row
# has already passed the bounded window and is forced aged-out above.
"age_days": age_days,
"reason": reason,
"coregistrants": list(coregistrants or []),
}