"""Actionability gate for incomplete SEC fundamentals.""" from __future__ import annotations import json from sqlalchemy import exists, select from sqlalchemy.ext.asyncio import AsyncSession from app.models.data_import_run import DataImportRun from app.models.fundamental_snapshot import FundamentalSnapshot from app.models.sec_filing_gap import SecFilingGap from app.models.ticker import Ticker from app.services import fundamental_data_refresh_service async def blocked_ciks(db: AsyncSession) -> set[str]: """CIKs whose SEC inputs are known incomplete. The durable queue covers filings already promoted around. The latest validation payload covers young filings still in the deferred retry window and new registrants with no XBRL history. """ if not await fundamental_data_refresh_service.is_enabled(db): return set() active = ( await db.execute( select(SecFilingGap.cik).where( ~exists().where( FundamentalSnapshot.accession == SecFilingGap.accession ) ) ) ).scalars().all() blocked = set(active) # Migration bootstrap: before the first scheduled import has materialized # the durable queue, recover unresolved promoted gaps from the import audit. # Once the queue has rows, the importer owns this state and this history scan # is no longer needed on setup reads. if not active and not await retry_queue_initialized(db): blocked.update(await _historical_unresolved_ciks(db)) payload = ( await db.execute( select(DataImportRun.validation_json) .where( DataImportRun.source == "sec_facts", DataImportRun.validation_json.is_not(None), ) .order_by(DataImportRun.id.desc()) .limit(1) ) ).scalar_one_or_none() if not payload: return blocked try: summary = json.loads(payload) except (TypeError, ValueError): return blocked for item in summary.get("missing_xbrl") or []: if item.get("cik"): blocked.add(str(item["cik"])) for item in summary.get("no_xbrl_filings") or []: if item.get("cik"): blocked.add(str(item["cik"])) return blocked async def retry_queue_initialized(db: AsyncSession) -> bool: """Whether a promoted run has synchronized the durable retry queue.""" payload = ( await db.execute( select(DataImportRun.row_counts_json) .where( DataImportRun.source == "sec_facts", DataImportRun.status == "promoted", DataImportRun.row_counts_json.is_not(None), ) .order_by(DataImportRun.id.desc()) .limit(1) ) ).scalar_one_or_none() if not payload: return False try: counts = json.loads(payload) except (TypeError, ValueError): return False return "retry_queue_added" in counts async def _historical_unresolved_ciks(db: AsyncSession) -> set[str]: payloads = ( await db.execute( select(DataImportRun.validation_json).where( DataImportRun.source == "sec_facts", DataImportRun.status == "promoted", DataImportRun.validation_json.is_not(None), ) ) ).scalars().all() accession_to_cik: dict[str, str] = {} for payload in payloads: try: summary = json.loads(payload) except (TypeError, ValueError): continue for item in summary.get("missing_xbrl") or []: accession = item.get("accession") cik = item.get("cik") if accession and cik: accession_to_cik[str(accession)] = str(cik) if not accession_to_cik: return set() resolved = set( ( await db.execute( select(FundamentalSnapshot.accession).where( FundamentalSnapshot.accession.in_(accession_to_cik) ) ) ).scalars() ) return { cik for accession, cik in accession_to_cik.items() if accession not in resolved } async def blocked_ticker_ids(db: AsyncSession) -> set[int]: ciks = await blocked_ciks(db) if not ciks: return set() rows = await db.execute(select(Ticker.id).where(Ticker.cik.in_(ciks))) return {int(ticker_id) for ticker_id in rows.scalars()} async def ticker_is_eligible(db: AsyncSession, ticker_id: int) -> bool: return ticker_id not in await blocked_ticker_ids(db)