dennisthiessen/signal-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: make SEC quality gating terminal-safe

Browse Source
This commit is contained in:
dennisthiessen
2026-08-04 07:58:18 +02:00
parent d1caac86b5
commit 4c0c0579f5
14 changed files with 629 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/fundamentals_quality_service.py
+85 -90
View File
@@ -3,8 +3,9 @@
from __future__ import annotations
import json
from dataclasses import dataclass
from sqlalchemy import exists, select
from sqlalchemy import exists, or_, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.data_import_run import DataImportRun
@@ -13,35 +14,41 @@ from app.models.sec_filing_gap import SecFilingGap
from app.models.ticker import Ticker
from app.services import fundamental_data_refresh_service
_SEC_FORMS = ("10-K", "10-Q", "10-K/A", "10-Q/A")
async def blocked_ciks(db: AsyncSession) -> set[str]:
"""CIKs whose SEC inputs are known incomplete.
The durable queue covers filings already promoted around. The latest
validation payload covers young filings still in the deferred retry window
and new registrants with no XBRL history.
"""
if not await fundamental_data_refresh_service.is_enabled(db):
return set()
@dataclass(frozen=True)
class SetupQuality:
eligible: bool
code: str | None = None
message: str | None = None
active = (
await db.execute(
select(SecFilingGap.cik).where(
~exists().where(
FundamentalSnapshot.accession == SecFilingGap.accession
)
)
)
).scalars().all()
blocked = set(active)
# Migration bootstrap: before the first scheduled import has materialized
# the durable queue, recover unresolved promoted gaps from the import audit.
# Once the queue has rows, the importer owns this state and this history scan
# is no longer needed on setup reads.
if not active and not await retry_queue_initialized(db):
blocked.update(await _historical_unresolved_ciks(db))
async def active_gaps(
db: AsyncSession,
ciks: set[str] | None = None,
) -> list[SecFilingGap]:
"""Unresolved gaps that have not been superseded by a later filing."""
matching_snapshot = exists().where(
FundamentalSnapshot.accession == SecFilingGap.accession
)
later_snapshot = exists().where(
FundamentalSnapshot.cik == SecFilingGap.cik,
FundamentalSnapshot.form.in_(_SEC_FORMS),
FundamentalSnapshot.filed_date > SecFilingGap.index_date,
)
stmt = select(SecFilingGap).where(
~matching_snapshot,
or_(SecFilingGap.index_date.is_(None), ~later_snapshot),
)
if ciks is not None:
if not ciks:
return []
stmt = stmt.where(SecFilingGap.cik.in_(ciks))
return list((await db.execute(stmt)).scalars().all())
async def _latest_validation(db: AsyncSession) -> dict:
payload = (
await db.execute(
select(DataImportRun.validation_json)
@@ -54,81 +61,38 @@ async def blocked_ciks(db: AsyncSession) -> set[str]:
)
).scalar_one_or_none()
if not payload:
return blocked
return {}
try:
summary = json.loads(payload)
except (TypeError, ValueError):
return blocked
return {}
return summary if isinstance(summary, dict) else {}
async def blocked_reasons_by_cik(db: AsyncSession) -> dict[str, str]:
"""Current SEC blocker code by CIK; no historical audit scan."""
if not await fundamental_data_refresh_service.is_enabled(db):
return {}
reasons = {gap.cik: "sec_filing_gap" for gap in await active_gaps(db)}
summary = await _latest_validation(db)
# New summaries carry the complete compact CIK set while the detailed lists
# stay capped for audit readability. Detailed entries supply the reason.
for cik in summary.get("setup_blocked_ciks") or []:
if cik:
reasons.setdefault(str(cik), "sec_filing_gap")
for item in summary.get("missing_xbrl") or []:
if item.get("cik"):
blocked.add(str(item["cik"]))
reasons.setdefault(str(item["cik"]), "sec_filing_gap")
for item in summary.get("no_xbrl_filings") or []:
if item.get("cik"):
blocked.add(str(item["cik"]))
return blocked
reasons[str(item["cik"])] = "no_xbrl_filings"
return reasons
async def retry_queue_initialized(db: AsyncSession) -> bool:
"""Whether a promoted run has synchronized the durable retry queue."""
payload = (
await db.execute(
select(DataImportRun.row_counts_json)
.where(
DataImportRun.source == "sec_facts",
DataImportRun.status == "promoted",
DataImportRun.row_counts_json.is_not(None),
)
.order_by(DataImportRun.id.desc())
.limit(1)
)
).scalar_one_or_none()
if not payload:
return False
try:
counts = json.loads(payload)
except (TypeError, ValueError):
return False
return "retry_queue_added" in counts
async def _historical_unresolved_ciks(db: AsyncSession) -> set[str]:
payloads = (
await db.execute(
select(DataImportRun.validation_json).where(
DataImportRun.source == "sec_facts",
DataImportRun.status == "promoted",
DataImportRun.validation_json.is_not(None),
)
)
).scalars().all()
accession_to_cik: dict[str, str] = {}
for payload in payloads:
try:
summary = json.loads(payload)
except (TypeError, ValueError):
continue
for item in summary.get("missing_xbrl") or []:
accession = item.get("accession")
cik = item.get("cik")
if accession and cik:
accession_to_cik[str(accession)] = str(cik)
if not accession_to_cik:
return set()
resolved = set(
(
await db.execute(
select(FundamentalSnapshot.accession).where(
FundamentalSnapshot.accession.in_(accession_to_cik)
)
)
).scalars()
)
return {
cik for accession, cik in accession_to_cik.items() if accession not in resolved
}
async def blocked_ciks(db: AsyncSession) -> set[str]:
return set(await blocked_reasons_by_cik(db))
async def blocked_ticker_ids(db: AsyncSession) -> set[int]:
@@ -139,5 +103,36 @@ async def blocked_ticker_ids(db: AsyncSession) -> set[int]:
return {int(ticker_id) for ticker_id in rows.scalars()}
async def ticker_quality(db: AsyncSession, symbol: str) -> SetupQuality:
ticker = (
await db.execute(
select(Ticker).where(Ticker.symbol == symbol.strip().upper())
)
).scalar_one_or_none()
if ticker is None or not ticker.cik:
return SetupQuality(eligible=True)
reason = (await blocked_reasons_by_cik(db)).get(ticker.cik)
if reason == "no_xbrl_filings":
return SetupQuality(
eligible=False,
code=reason,
message=(
"No SEC 10-K/10-Q is available for this registrant, so new setups "
"are paused. New registrants clear automatically after their first "
"filing; a successor shell needs an SEC CIK override."
),
)
if reason:
return SetupQuality(
eligible=False,
code=reason,
message=(
"A recent SEC filing is still being reconciled, so new setups are "
"paused. The scheduled fundamentals import retries it automatically."
),
)
return SetupQuality(eligible=True)
async def ticker_is_eligible(db: AsyncSession, ticker_id: int) -> bool:
return ticker_id not in await blocked_ticker_ids(db)
app/services/rr_scanner_service.py
+21 -1
View File
@@ -27,7 +27,7 @@ from app.models.signal_context_snapshot import SignalContextSnapshot
from app.models.ticker import Ticker
from app.models.trade_setup import TradeSetup
from app.services.indicator_service import _extract_ohlcv, compute_atr
from app.services import fundamentals_quality_service
from app.services import fundamentals_quality_service, system_event_service
from app.services.price_service import query_ohlcv
from app.services.qualification import setup_qualifies
from app.services.sr_service import detect_gate_target_ladder
@@ -750,6 +750,16 @@ async def scan_all_tickers(
logger.exception(
"Could not resolve fundamentals quality; blocking this scan closed"
)
await system_event_service.log_event_standalone(
severity="error",
source="rr_scanner",
code="fundamentals_quality_unavailable",
message=(
"The fundamentals quality gate could not be evaluated; the "
"universe scan was blocked to avoid issuing unchecked setups."
),
dedup_key="rr_scanner:fundamentals_quality_unavailable",
)
fundamentals_blocked_ids = {ticker_id for ticker_id, _ in ticker_rows}
# Gate-reset observations must use the same runtime activation settings as
@@ -924,6 +934,16 @@ async def get_trade_setups(
logger.exception(
"Could not resolve fundamentals quality; hiding actionable setups"
)
await system_event_service.log_event_standalone(
severity="error",
source="rr_scanner",
code="fundamentals_quality_unavailable",
message=(
"The fundamentals quality gate could not be evaluated; actionable "
"setups were hidden until the metadata check recovers."
),
dedup_key="rr_scanner:fundamentals_quality_unavailable",
)
return []
if exclude_open_trade_tickers:
# Manual book only. The shadow book holds the *top-ranked* names by
app/services/sec_fundamentals_importer.py
+98 -73
View File
@@ -81,6 +81,7 @@ MIN_BACKFILL_COVERAGE = 0.5
# three); past that it is misfiled, not late, and blocking forever costs more
# than the missing filing does — see the unresolved-filing guardrail below.
MISSING_XBRL_RETRY_DAYS = 3
FILING_GAP_ESCALATE_DAYS = 14
# Share-count band a co-registrant-recovered row must land in, relative to the
# issuer's own last snapshot. Wide enough for buybacks/issuance, nowhere near
# wide enough to let a subsidiary shell's token float through (see _shares_continuous).
@@ -154,7 +155,6 @@ class SecFundamentalsImporter:
# of a combined filing). Only populated for accessions a tracked issuer filed.
self._coregistrants: dict[str, list[int]] = {}
self._retry_rows: list[dict[str, Any]] = []
self._retry_queue_bootstrap_pending = False
self._latest_index_date: date | None = None
self._backfill = False
@@ -185,18 +185,14 @@ class SecFundamentalsImporter:
)
self._retry_rows = []
if not self._backfill:
self._retry_queue_bootstrap_pending = not (
await fundamentals_quality_service.retry_queue_initialized(db)
)
self._retry_rows = await self._retry_backlog(
db,
set(self._resolved.cik_to_ticker_ids),
include_history=self._retry_queue_bootstrap_pending,
)
# Company Facts can change while the daily index revision stays fixed.
# Returning None deliberately bypasses the framework's no-op gate so a
# scheduled run retries every active gap.
return None if self._retry_rows or self._retry_queue_bootstrap_pending else revision
return None if self._retry_rows else revision
async def stage(self, db) -> StagedFundamentals:
assert self._resolved is not None, "detect_revision must run first"
@@ -211,10 +207,9 @@ class SecFundamentalsImporter:
if r["cik"] in cik_to_tids:
filed_by_cik[r["cik"]].append(r)
# Promoted-around filings live in a small durable retry queue. Historical
# validation payloads bootstrap gaps created before the queue existed.
# Merge them into the normal incremental work so the scheduled import,
# not an operator-run full reparse, heals them when SEC catches up.
# Promoted-around filings live in a small durable retry queue, including
# the one-time migration backfill. Merge them into normal incremental
# work so the scheduled importer heals them without operator action.
if not self._backfill:
seen = {
(int(cik), row["accession"])
@@ -286,6 +281,9 @@ class SecFundamentalsImporter:
fiscal_year_end = sub.get("fiscal_year_end")
recovered_rows: list[SnapshotRow] = []
index_rows = {
row["accession"]: row for row in filed_by_cik.get(cik, [])
}
if is_backfill:
accns = set(xbrl_meta)
else:
@@ -340,6 +338,19 @@ class SecFundamentalsImporter:
# fiscalYearEnd (MMDD) is what lets the parser derive period identity from
# reportDate instead of SEC's unreliable fy/fp fields.
result = parser.parse_snapshots(cf, xbrl_meta, accns, fiscal_year_end=fiscal_year_end)
for skipped in result.skipped_filings:
index_row = index_rows.get(skipped["accession"])
if index_row is not None:
# Facts are present but our parser cannot construct a snapshot.
# This is terminal for this run: promote around it immediately,
# keep the issuer blocked, and retry/escalate through the queue.
staged.missing_xbrl.append(_missing(
cik,
{**index_row, "_retry_queue": True},
"parser_unusable",
self.today,
self._coregistrants.get(skipped["accession"]),
))
staged.rows.extend(result.rows)
staged.rows.extend(recovered_rows)
staged.skipped_filings.extend(result.skipped_filings)
@@ -442,13 +453,19 @@ class SecFundamentalsImporter:
"skipped_filings": len(staged.skipped_filings),
"field_issues": len(staged.field_issues),
"skipped_non_xbrl": len(staged.skipped_non_xbrl),
"no_xbrl_filings": staged.no_xbrl_filings,
"no_xbrl_filings": staged.no_xbrl_filings[:50],
"no_xbrl_filings_count": len(staged.no_xbrl_filings),
"missing_xbrl": staged.missing_xbrl,
"missing_xbrl": staged.missing_xbrl[:50],
"missing_xbrl_count": len(staged.missing_xbrl),
"missing_xbrl_blocking": len(blocking),
"recovered_from_coregistrant": staged.recovered,
"recovered_from_coregistrant": staged.recovered[:50],
"recovered_count": len(staged.recovered),
# Complete compact gate input; detailed audit lists above stay capped.
"setup_blocked_ciks": sorted({
str(item["cik"])
for item in [*staged.missing_xbrl, *staged.no_xbrl_filings]
if item.get("cik")
}),
"invalid_payloads": staged.invalid_payloads,
"cik_updates": len(staged.resolved.cik_updates),
# differing existing accessions (immutable — kept, reported here)
@@ -508,13 +525,15 @@ class SecFundamentalsImporter:
await db.execute(stmt)
inserted += 1
# Synchronize the active retry queue in the same transaction as snapshot
# promotion. Reconstructed accessions leave the queue; aged-out gaps enter
# or refresh it and will be retried by the next scheduled import.
existing_gap_accessions = set(
(await db.execute(select(SecFilingGap.accession))).scalars().all()
)
# Synchronize the retry queue in the snapshot-promotion transaction.
existing_gaps = (await db.execute(select(SecFilingGap))).scalars().all()
existing_gap_accessions = {gap.accession for gap in existing_gaps}
resolved_accessions = {row.accession for row in staged.rows}
# A filing now classified non-XBRL can never yield a snapshot and is no
# longer a fundamentals completeness gap.
resolved_accessions.update(
item["accession"] for item in staged.skipped_non_xbrl
)
queue_resolved = 0
if resolved_accessions:
result = await db.execute(
@@ -524,8 +543,8 @@ class SecFundamentalsImporter:
)
queue_resolved = int(result.rowcount or 0)
tolerated = _past_retry_window(staged.missing_xbrl)
now = _now()
tolerated = _past_retry_window(staged.missing_xbrl)
for gap in tolerated:
stmt = insert_for_session(db, SecFilingGap).values(
cik=gap["cik"],
@@ -550,6 +569,20 @@ class SecFundamentalsImporter:
},
)
)
# Remove gaps made irrelevant by a later valid 10-K/10-Q. Quality reads
# already ignore them; physical cleanup keeps the queue small.
active_ids = {gap.id for gap in await fundamentals_quality_service.active_gaps(db)}
obsolete_ids = {
gap.id for gap in existing_gaps
if gap.id not in active_ids and gap.accession not in resolved_accessions
}
if obsolete_ids:
result = await db.execute(
delete(SecFilingGap).where(SecFilingGap.id.in_(obsolete_ids))
)
queue_resolved += int(result.rowcount or 0)
newly_queued = [
gap for gap in tolerated
if gap["accession"] not in existing_gap_accessions
@@ -574,6 +607,40 @@ class SecFundamentalsImporter:
created_at=_now(),
))
# Persistent current gaps get one actionable escalation rather than a
# daily warning. The nullable marker makes this durable and noise-free.
escalation_cutoff = now - timedelta(days=FILING_GAP_ESCALATE_DAYS)
aged_gaps = (
await db.execute(
select(SecFilingGap).where(
SecFilingGap.first_seen_at <= escalation_cutoff,
SecFilingGap.escalated_at.is_(None),
)
)
).scalars().all()
if aged_gaps:
named = ", ".join(
f"{gap.cik}/{gap.accession} ({gap.reason})"
for gap in aged_gaps[:10]
)
db.add(SystemEvent(
severity="warning",
source="sec_facts",
code="filing_gap_aged",
message=(
f"{len(aged_gaps)} SEC filing gap(s) remain unresolved after "
f"{FILING_GAP_ESCALATE_DAYS} days; affected setups remain paused. "
f"Review the filing/CIK mapping or parser: {named}"
)[:4000],
dedup_key=f"sec_facts:filing_gap_aged:{run_id}",
created_at=now,
))
await db.execute(
update(SecFilingGap)
.where(SecFilingGap.id.in_([gap.id for gap in aged_gaps]))
.values(escalated_at=now)
)
# Recovered rows are real data from an unexpected place — record where they
# came from, so a wrong recovery is auditable rather than invisible.
if staged.recovered:
@@ -641,20 +708,14 @@ class SecFundamentalsImporter:
self,
db,
tracked_ciks: set[int],
*,
include_history: bool,
) -> list[dict[str, Any]]:
"""Active gaps plus pre-queue gaps from promoted validation history."""
"""Active typed gaps; migration 028 owns historical bootstrap."""
if not tracked_ciks:
return []
tracked = {cik10(cik) for cik in tracked_ciks}
candidates: dict[str, dict[str, Any]] = {}
queued = (
await db.execute(
select(SecFilingGap).where(SecFilingGap.cik.in_(tracked))
)
).scalars().all()
queued = await fundamentals_quality_service.active_gaps(db, tracked)
for gap in queued:
try:
coregistrants = json.loads(gap.coregistrant_ciks_json or "[]")
@@ -667,51 +728,9 @@ class SecFundamentalsImporter:
"index_date": gap.index_date,
"reason": gap.reason,
"coregistrants": coregistrants,
"_retry_queue": True,
}
# Bootstrap warnings produced before sec_filing_gaps existed. Promoted
# runs contain only aged-out gaps; deferred/failed runs are naturally
# retried because source_max_date has not advanced.
if include_history:
histories = (
await db.execute(
select(DataImportRun.validation_json)
.where(
DataImportRun.source == SOURCE,
DataImportRun.status == STATUS_PROMOTED,
DataImportRun.validation_json.is_not(None),
)
.order_by(DataImportRun.id.asc())
)
).scalars().all()
for payload in histories:
try:
summary = json.loads(payload)
except (TypeError, ValueError):
continue
for item in summary.get("missing_xbrl") or []:
accession = item.get("accession")
cik = str(item.get("cik") or "")
if not accession or cik not in tracked or accession in candidates:
continue
raw_date = item.get("index_date")
try:
index_date = (
date.fromisoformat(raw_date)
if isinstance(raw_date, str)
else raw_date
)
except ValueError:
index_date = None
candidates[accession] = {
"cik": cik,
"accession": accession,
"form": item.get("form"),
"index_date": index_date,
"reason": item.get("reason") or "not_in_companyfacts",
"coregistrants": item.get("coregistrants") or [],
}
if not candidates:
return []
resolved = set(
@@ -848,13 +867,19 @@ def _missing(
up by hand (EDGAR accession + the index date it was seen on) and to decide
whether it is still young enough to be worth blocking on."""
index_date = row.get("index_date")
age_days = (
(today - index_date).days if isinstance(index_date, date) else 0
)
if row.get("_retry_queue"):
age_days = max(age_days, MISSING_XBRL_RETRY_DAYS + 1)
return {
"cik": cik10(cik),
"accession": row["accession"],
"form": row.get("form"),
"index_date": index_date,
# No index date (older cached rows) => age 0 => blocks, the safe default.
"age_days": (today - index_date).days if isinstance(index_date, date) else 0,
# A newly observed row without a date blocks safely. A durable queue row
# has already passed the bounded window and is forced aged-out above.
"age_days": age_days,
"reason": reason,
"coregistrants": list(coregistrants or []),
}