fix: gate setups on SEC filing completeness

2026-08-03 23:47:07 +02:00
parent 7d703ea524
commit 3a6900d45a
11 changed files with 723 additions and 56 deletions
@@ -0,0 +1,143 @@
+"""Actionability gate for incomplete SEC fundamentals."""
+
+from __future__ import annotations
+
+import json
+
+from sqlalchemy import exists, select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.models.data_import_run import DataImportRun
+from app.models.fundamental_snapshot import FundamentalSnapshot
+from app.models.sec_filing_gap import SecFilingGap
+from app.models.ticker import Ticker
+from app.services import fundamental_data_refresh_service
+
+
+async def blocked_ciks(db: AsyncSession) -> set[str]:
+    """CIKs whose SEC inputs are known incomplete.
+
+    The durable queue covers filings already promoted around. The latest
+    validation payload covers young filings still in the deferred retry window
+    and new registrants with no XBRL history.
+    """
+    if not await fundamental_data_refresh_service.is_enabled(db):
+        return set()
+
+    active = (
+        await db.execute(
+            select(SecFilingGap.cik).where(
+                ~exists().where(
+                    FundamentalSnapshot.accession == SecFilingGap.accession
+                )
+            )
+        )
+    ).scalars().all()
+    blocked = set(active)
+
+    # Migration bootstrap: before the first scheduled import has materialized
+    # the durable queue, recover unresolved promoted gaps from the import audit.
+    # Once the queue has rows, the importer owns this state and this history scan
+    # is no longer needed on setup reads.
+    if not active and not await retry_queue_initialized(db):
+        blocked.update(await _historical_unresolved_ciks(db))
+
+    payload = (
+        await db.execute(
+            select(DataImportRun.validation_json)
+            .where(
+                DataImportRun.source == "sec_facts",
+                DataImportRun.validation_json.is_not(None),
+            )
+            .order_by(DataImportRun.id.desc())
+            .limit(1)
+        )
+    ).scalar_one_or_none()
+    if not payload:
+        return blocked
+    try:
+        summary = json.loads(payload)
+    except (TypeError, ValueError):
+        return blocked
+
+    for item in summary.get("missing_xbrl") or []:
+        if item.get("cik"):
+            blocked.add(str(item["cik"]))
+    for item in summary.get("no_xbrl_filings") or []:
+        if item.get("cik"):
+            blocked.add(str(item["cik"]))
+    return blocked
+
+
+async def retry_queue_initialized(db: AsyncSession) -> bool:
+    """Whether a promoted run has synchronized the durable retry queue."""
+    payload = (
+        await db.execute(
+            select(DataImportRun.row_counts_json)
+            .where(
+                DataImportRun.source == "sec_facts",
+                DataImportRun.status == "promoted",
+                DataImportRun.row_counts_json.is_not(None),
+            )
+            .order_by(DataImportRun.id.desc())
+            .limit(1)
+        )
+    ).scalar_one_or_none()
+    if not payload:
+        return False
+    try:
+        counts = json.loads(payload)
+    except (TypeError, ValueError):
+        return False
+    return "retry_queue_added" in counts
+
+
+async def _historical_unresolved_ciks(db: AsyncSession) -> set[str]:
+    payloads = (
+        await db.execute(
+            select(DataImportRun.validation_json).where(
+                DataImportRun.source == "sec_facts",
+                DataImportRun.status == "promoted",
+                DataImportRun.validation_json.is_not(None),
+            )
+        )
+    ).scalars().all()
+
+    accession_to_cik: dict[str, str] = {}
+    for payload in payloads:
+        try:
+            summary = json.loads(payload)
+        except (TypeError, ValueError):
+            continue
+        for item in summary.get("missing_xbrl") or []:
+            accession = item.get("accession")
+            cik = item.get("cik")
+            if accession and cik:
+                accession_to_cik[str(accession)] = str(cik)
+
+    if not accession_to_cik:
+        return set()
+    resolved = set(
+        (
+            await db.execute(
+                select(FundamentalSnapshot.accession).where(
+                    FundamentalSnapshot.accession.in_(accession_to_cik)
+                )
+            )
+        ).scalars()
+    )
+    return {
+        cik for accession, cik in accession_to_cik.items() if accession not in resolved
+    }
+
+
+async def blocked_ticker_ids(db: AsyncSession) -> set[int]:
+    ciks = await blocked_ciks(db)
+    if not ciks:
+        return set()
+    rows = await db.execute(select(Ticker.id).where(Ticker.cik.in_(ciks)))
+    return {int(ticker_id) for ticker_id in rows.scalars()}
+
+
+async def ticker_is_eligible(db: AsyncSession, ticker_id: int) -> bool:
+    return ticker_id not in await blocked_ticker_ids(db)