fix: close remaining ingestion review gaps
This commit is contained in:
@@ -5,7 +5,7 @@ from __future__ import annotations
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
|
||||
from sqlalchemy import exists, or_, select
|
||||
from sqlalchemy import exists, func, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.models.data_import_run import DataImportRun
|
||||
@@ -32,14 +32,18 @@ async def active_gaps(
|
||||
matching_snapshot = exists().where(
|
||||
FundamentalSnapshot.accession == SecFilingGap.accession
|
||||
)
|
||||
gap_date = func.coalesce(
|
||||
SecFilingGap.index_date,
|
||||
func.date(SecFilingGap.first_seen_at),
|
||||
)
|
||||
later_snapshot = exists().where(
|
||||
FundamentalSnapshot.cik == SecFilingGap.cik,
|
||||
FundamentalSnapshot.form.in_(_SEC_FORMS),
|
||||
FundamentalSnapshot.filed_date > SecFilingGap.index_date,
|
||||
FundamentalSnapshot.filed_date > gap_date,
|
||||
)
|
||||
stmt = select(SecFilingGap).where(
|
||||
~matching_snapshot,
|
||||
or_(SecFilingGap.index_date.is_(None), ~later_snapshot),
|
||||
~later_snapshot,
|
||||
)
|
||||
if ciks is not None:
|
||||
if not ciks:
|
||||
|
||||
@@ -342,11 +342,12 @@ class SecFundamentalsImporter:
|
||||
index_row = index_rows.get(skipped["accession"])
|
||||
if index_row is not None:
|
||||
# Facts are present but our parser cannot construct a snapshot.
|
||||
# This is terminal for this run: promote around it immediately,
|
||||
# keep the issuer blocked, and retry/escalate through the queue.
|
||||
# A new index row keeps the normal grace period before promotion;
|
||||
# a row already read from the queue retains its _retry_queue marker
|
||||
# so later imports promote and retry without wedging the index.
|
||||
staged.missing_xbrl.append(_missing(
|
||||
cik,
|
||||
{**index_row, "_retry_queue": True},
|
||||
index_row,
|
||||
"parser_unusable",
|
||||
self.today,
|
||||
self._coregistrants.get(skipped["accession"]),
|
||||
@@ -488,7 +489,10 @@ class SecFundamentalsImporter:
|
||||
retryable=(
|
||||
len(messages) == 1
|
||||
and bool(blocking)
|
||||
and all(m.get("reason") == "not_in_companyfacts" for m in blocking)
|
||||
and all(
|
||||
m.get("reason") in {"not_in_companyfacts", "parser_unusable"}
|
||||
for m in blocking
|
||||
)
|
||||
),
|
||||
deferred_alert_after_days=MISSING_XBRL_RETRY_DAYS,
|
||||
deferred_alert_messages=(
|
||||
|
||||
Reference in New Issue
Block a user