fix: make SEC quality gating terminal-safe
This commit is contained in:
@@ -3,8 +3,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import exists, select
|
||||
from sqlalchemy import exists, or_, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.data_import_run import DataImportRun
|
||||
@@ -13,35 +14,41 @@ from app.models.sec_filing_gap import SecFilingGap
|
||||
from app.models.ticker import Ticker
|
||||
from app.services import fundamental_data_refresh_service
|
||||
|
||||
_SEC_FORMS = ("10-K", "10-Q", "10-K/A", "10-Q/A")
|
||||
|
||||
async def blocked_ciks(db: AsyncSession) -> set[str]:
|
||||
"""CIKs whose SEC inputs are known incomplete.
|
||||
|
||||
The durable queue covers filings already promoted around. The latest
|
||||
validation payload covers young filings still in the deferred retry window
|
||||
and new registrants with no XBRL history.
|
||||
"""
|
||||
if not await fundamental_data_refresh_service.is_enabled(db):
|
||||
return set()
|
||||
@dataclass(frozen=True)
|
||||
class SetupQuality:
|
||||
eligible: bool
|
||||
code: str | None = None
|
||||
message: str | None = None
|
||||
|
||||
active = (
|
||||
await db.execute(
|
||||
select(SecFilingGap.cik).where(
|
||||
~exists().where(
|
||||
FundamentalSnapshot.accession == SecFilingGap.accession
|
||||
)
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
blocked = set(active)
|
||||
|
||||
# Migration bootstrap: before the first scheduled import has materialized
|
||||
# the durable queue, recover unresolved promoted gaps from the import audit.
|
||||
# Once the queue has rows, the importer owns this state and this history scan
|
||||
# is no longer needed on setup reads.
|
||||
if not active and not await retry_queue_initialized(db):
|
||||
blocked.update(await _historical_unresolved_ciks(db))
|
||||
async def active_gaps(
|
||||
db: AsyncSession,
|
||||
ciks: set[str] | None = None,
|
||||
) -> list[SecFilingGap]:
|
||||
"""Unresolved gaps that have not been superseded by a later filing."""
|
||||
matching_snapshot = exists().where(
|
||||
FundamentalSnapshot.accession == SecFilingGap.accession
|
||||
)
|
||||
later_snapshot = exists().where(
|
||||
FundamentalSnapshot.cik == SecFilingGap.cik,
|
||||
FundamentalSnapshot.form.in_(_SEC_FORMS),
|
||||
FundamentalSnapshot.filed_date > SecFilingGap.index_date,
|
||||
)
|
||||
stmt = select(SecFilingGap).where(
|
||||
~matching_snapshot,
|
||||
or_(SecFilingGap.index_date.is_(None), ~later_snapshot),
|
||||
)
|
||||
if ciks is not None:
|
||||
if not ciks:
|
||||
return []
|
||||
stmt = stmt.where(SecFilingGap.cik.in_(ciks))
|
||||
return list((await db.execute(stmt)).scalars().all())
|
||||
|
||||
|
||||
async def _latest_validation(db: AsyncSession) -> dict:
|
||||
payload = (
|
||||
await db.execute(
|
||||
select(DataImportRun.validation_json)
|
||||
@@ -54,81 +61,38 @@ async def blocked_ciks(db: AsyncSession) -> set[str]:
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not payload:
|
||||
return blocked
|
||||
return {}
|
||||
try:
|
||||
summary = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
return blocked
|
||||
return {}
|
||||
return summary if isinstance(summary, dict) else {}
|
||||
|
||||
|
||||
async def blocked_reasons_by_cik(db: AsyncSession) -> dict[str, str]:
|
||||
"""Current SEC blocker code by CIK; no historical audit scan."""
|
||||
if not await fundamental_data_refresh_service.is_enabled(db):
|
||||
return {}
|
||||
|
||||
reasons = {gap.cik: "sec_filing_gap" for gap in await active_gaps(db)}
|
||||
summary = await _latest_validation(db)
|
||||
|
||||
# New summaries carry the complete compact CIK set while the detailed lists
|
||||
# stay capped for audit readability. Detailed entries supply the reason.
|
||||
for cik in summary.get("setup_blocked_ciks") or []:
|
||||
if cik:
|
||||
reasons.setdefault(str(cik), "sec_filing_gap")
|
||||
for item in summary.get("missing_xbrl") or []:
|
||||
if item.get("cik"):
|
||||
blocked.add(str(item["cik"]))
|
||||
reasons.setdefault(str(item["cik"]), "sec_filing_gap")
|
||||
for item in summary.get("no_xbrl_filings") or []:
|
||||
if item.get("cik"):
|
||||
blocked.add(str(item["cik"]))
|
||||
return blocked
|
||||
reasons[str(item["cik"])] = "no_xbrl_filings"
|
||||
return reasons
|
||||
|
||||
|
||||
async def retry_queue_initialized(db: AsyncSession) -> bool:
|
||||
"""Whether a promoted run has synchronized the durable retry queue."""
|
||||
payload = (
|
||||
await db.execute(
|
||||
select(DataImportRun.row_counts_json)
|
||||
.where(
|
||||
DataImportRun.source == "sec_facts",
|
||||
DataImportRun.status == "promoted",
|
||||
DataImportRun.row_counts_json.is_not(None),
|
||||
)
|
||||
.order_by(DataImportRun.id.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if not payload:
|
||||
return False
|
||||
try:
|
||||
counts = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return "retry_queue_added" in counts
|
||||
|
||||
|
||||
async def _historical_unresolved_ciks(db: AsyncSession) -> set[str]:
|
||||
payloads = (
|
||||
await db.execute(
|
||||
select(DataImportRun.validation_json).where(
|
||||
DataImportRun.source == "sec_facts",
|
||||
DataImportRun.status == "promoted",
|
||||
DataImportRun.validation_json.is_not(None),
|
||||
)
|
||||
)
|
||||
).scalars().all()
|
||||
|
||||
accession_to_cik: dict[str, str] = {}
|
||||
for payload in payloads:
|
||||
try:
|
||||
summary = json.loads(payload)
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
for item in summary.get("missing_xbrl") or []:
|
||||
accession = item.get("accession")
|
||||
cik = item.get("cik")
|
||||
if accession and cik:
|
||||
accession_to_cik[str(accession)] = str(cik)
|
||||
|
||||
if not accession_to_cik:
|
||||
return set()
|
||||
resolved = set(
|
||||
(
|
||||
await db.execute(
|
||||
select(FundamentalSnapshot.accession).where(
|
||||
FundamentalSnapshot.accession.in_(accession_to_cik)
|
||||
)
|
||||
)
|
||||
).scalars()
|
||||
)
|
||||
return {
|
||||
cik for accession, cik in accession_to_cik.items() if accession not in resolved
|
||||
}
|
||||
async def blocked_ciks(db: AsyncSession) -> set[str]:
|
||||
return set(await blocked_reasons_by_cik(db))
|
||||
|
||||
|
||||
async def blocked_ticker_ids(db: AsyncSession) -> set[int]:
|
||||
@@ -139,5 +103,36 @@ async def blocked_ticker_ids(db: AsyncSession) -> set[int]:
|
||||
return {int(ticker_id) for ticker_id in rows.scalars()}
|
||||
|
||||
|
||||
async def ticker_quality(db: AsyncSession, symbol: str) -> SetupQuality:
|
||||
ticker = (
|
||||
await db.execute(
|
||||
select(Ticker).where(Ticker.symbol == symbol.strip().upper())
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if ticker is None or not ticker.cik:
|
||||
return SetupQuality(eligible=True)
|
||||
reason = (await blocked_reasons_by_cik(db)).get(ticker.cik)
|
||||
if reason == "no_xbrl_filings":
|
||||
return SetupQuality(
|
||||
eligible=False,
|
||||
code=reason,
|
||||
message=(
|
||||
"No SEC 10-K/10-Q is available for this registrant, so new setups "
|
||||
"are paused. New registrants clear automatically after their first "
|
||||
"filing; a successor shell needs an SEC CIK override."
|
||||
),
|
||||
)
|
||||
if reason:
|
||||
return SetupQuality(
|
||||
eligible=False,
|
||||
code=reason,
|
||||
message=(
|
||||
"A recent SEC filing is still being reconciled, so new setups are "
|
||||
"paused. The scheduled fundamentals import retries it automatically."
|
||||
),
|
||||
)
|
||||
return SetupQuality(eligible=True)
|
||||
|
||||
|
||||
async def ticker_is_eligible(db: AsyncSession, ticker_id: int) -> bool:
|
||||
return ticker_id not in await blocked_ticker_ids(db)
|
||||
|
||||
Reference in New Issue
Block a user