144 lines
4.5 KiB
Python
144 lines
4.5 KiB
Python
"""Actionability gate for incomplete SEC fundamentals."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from sqlalchemy import exists, select
|
|
from sqlalchemy.ext.asyncio import AsyncSession
|
|
|
|
from app.models.data_import_run import DataImportRun
|
|
from app.models.fundamental_snapshot import FundamentalSnapshot
|
|
from app.models.sec_filing_gap import SecFilingGap
|
|
from app.models.ticker import Ticker
|
|
from app.services import fundamental_data_refresh_service
|
|
|
|
|
|
async def blocked_ciks(db: AsyncSession) -> set[str]:
|
|
"""CIKs whose SEC inputs are known incomplete.
|
|
|
|
The durable queue covers filings already promoted around. The latest
|
|
validation payload covers young filings still in the deferred retry window
|
|
and new registrants with no XBRL history.
|
|
"""
|
|
if not await fundamental_data_refresh_service.is_enabled(db):
|
|
return set()
|
|
|
|
active = (
|
|
await db.execute(
|
|
select(SecFilingGap.cik).where(
|
|
~exists().where(
|
|
FundamentalSnapshot.accession == SecFilingGap.accession
|
|
)
|
|
)
|
|
)
|
|
).scalars().all()
|
|
blocked = set(active)
|
|
|
|
# Migration bootstrap: before the first scheduled import has materialized
|
|
# the durable queue, recover unresolved promoted gaps from the import audit.
|
|
# Once the queue has rows, the importer owns this state and this history scan
|
|
# is no longer needed on setup reads.
|
|
if not active and not await retry_queue_initialized(db):
|
|
blocked.update(await _historical_unresolved_ciks(db))
|
|
|
|
payload = (
|
|
await db.execute(
|
|
select(DataImportRun.validation_json)
|
|
.where(
|
|
DataImportRun.source == "sec_facts",
|
|
DataImportRun.validation_json.is_not(None),
|
|
)
|
|
.order_by(DataImportRun.id.desc())
|
|
.limit(1)
|
|
)
|
|
).scalar_one_or_none()
|
|
if not payload:
|
|
return blocked
|
|
try:
|
|
summary = json.loads(payload)
|
|
except (TypeError, ValueError):
|
|
return blocked
|
|
|
|
for item in summary.get("missing_xbrl") or []:
|
|
if item.get("cik"):
|
|
blocked.add(str(item["cik"]))
|
|
for item in summary.get("no_xbrl_filings") or []:
|
|
if item.get("cik"):
|
|
blocked.add(str(item["cik"]))
|
|
return blocked
|
|
|
|
|
|
async def retry_queue_initialized(db: AsyncSession) -> bool:
|
|
"""Whether a promoted run has synchronized the durable retry queue."""
|
|
payload = (
|
|
await db.execute(
|
|
select(DataImportRun.row_counts_json)
|
|
.where(
|
|
DataImportRun.source == "sec_facts",
|
|
DataImportRun.status == "promoted",
|
|
DataImportRun.row_counts_json.is_not(None),
|
|
)
|
|
.order_by(DataImportRun.id.desc())
|
|
.limit(1)
|
|
)
|
|
).scalar_one_or_none()
|
|
if not payload:
|
|
return False
|
|
try:
|
|
counts = json.loads(payload)
|
|
except (TypeError, ValueError):
|
|
return False
|
|
return "retry_queue_added" in counts
|
|
|
|
|
|
async def _historical_unresolved_ciks(db: AsyncSession) -> set[str]:
|
|
payloads = (
|
|
await db.execute(
|
|
select(DataImportRun.validation_json).where(
|
|
DataImportRun.source == "sec_facts",
|
|
DataImportRun.status == "promoted",
|
|
DataImportRun.validation_json.is_not(None),
|
|
)
|
|
)
|
|
).scalars().all()
|
|
|
|
accession_to_cik: dict[str, str] = {}
|
|
for payload in payloads:
|
|
try:
|
|
summary = json.loads(payload)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
for item in summary.get("missing_xbrl") or []:
|
|
accession = item.get("accession")
|
|
cik = item.get("cik")
|
|
if accession and cik:
|
|
accession_to_cik[str(accession)] = str(cik)
|
|
|
|
if not accession_to_cik:
|
|
return set()
|
|
resolved = set(
|
|
(
|
|
await db.execute(
|
|
select(FundamentalSnapshot.accession).where(
|
|
FundamentalSnapshot.accession.in_(accession_to_cik)
|
|
)
|
|
)
|
|
).scalars()
|
|
)
|
|
return {
|
|
cik for accession, cik in accession_to_cik.items() if accession not in resolved
|
|
}
|
|
|
|
|
|
async def blocked_ticker_ids(db: AsyncSession) -> set[int]:
|
|
ciks = await blocked_ciks(db)
|
|
if not ciks:
|
|
return set()
|
|
rows = await db.execute(select(Ticker.id).where(Ticker.cik.in_(ciks)))
|
|
return {int(ticker_id) for ticker_id in rows.scalars()}
|
|
|
|
|
|
async def ticker_is_eligible(db: AsyncSession, ticker_id: int) -> bool:
|
|
return ticker_id not in await blocked_ticker_ids(db)
|