fix: close remaining ingestion review gaps

This commit is contained in:
2026-08-04 08:55:30 +02:00
parent 59ac108c90
commit f49b422095
6 changed files with 141 additions and 9 deletions
+7 -3
View File
@@ -5,7 +5,7 @@ from __future__ import annotations
import json
from dataclasses import dataclass
from sqlalchemy import exists, or_, select
from sqlalchemy import exists, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.models.data_import_run import DataImportRun
@@ -32,14 +32,18 @@ async def active_gaps(
matching_snapshot = exists().where(
FundamentalSnapshot.accession == SecFilingGap.accession
)
gap_date = func.coalesce(
SecFilingGap.index_date,
func.date(SecFilingGap.first_seen_at),
)
later_snapshot = exists().where(
FundamentalSnapshot.cik == SecFilingGap.cik,
FundamentalSnapshot.form.in_(_SEC_FORMS),
FundamentalSnapshot.filed_date > SecFilingGap.index_date,
FundamentalSnapshot.filed_date > gap_date,
)
stmt = select(SecFilingGap).where(
~matching_snapshot,
or_(SecFilingGap.index_date.is_(None), ~later_snapshot),
~later_snapshot,
)
if ciks is not None:
if not ciks:
+8 -4
View File
@@ -342,11 +342,12 @@ class SecFundamentalsImporter:
index_row = index_rows.get(skipped["accession"])
if index_row is not None:
# Facts are present but our parser cannot construct a snapshot.
# This is terminal for this run: promote around it immediately,
# keep the issuer blocked, and retry/escalate through the queue.
# A new index row keeps the normal grace period before promotion;
# a row already read from the queue retains its _retry_queue marker
# so later imports promote and retry without wedging the index.
staged.missing_xbrl.append(_missing(
cik,
{**index_row, "_retry_queue": True},
index_row,
"parser_unusable",
self.today,
self._coregistrants.get(skipped["accession"]),
@@ -488,7 +489,10 @@ class SecFundamentalsImporter:
retryable=(
len(messages) == 1
and bool(blocking)
and all(m.get("reason") == "not_in_companyfacts" for m in blocking)
and all(
m.get("reason") in {"not_in_companyfacts", "parser_unusable"}
for m in blocking
)
),
deferred_alert_after_days=MISSING_XBRL_RETRY_DAYS,
deferred_alert_messages=(