fix(sec): warn when deferred imports stay stale

app/services/data_import.py

@@ -30,7 +30,7 @@ import hashlib
 import json
 import logging
 from dataclasses import dataclass, field
-from datetime import date, datetime, timezone
+from datetime import date, datetime, timedelta, timezone
 from typing import Any, Protocol, runtime_checkable
 
 from sqlalchemy import select, text
@@ -69,9 +69,11 @@ class ValidationResult:
     summary: dict[str, Any] = field(default_factory=dict)
     source_max_date: date | None = None
     messages: list[str] = field(default_factory=list)
-    # Expected source-side lag: retry next run without an operational alert.
-    # This is only meaningful when ok=False.
+    # Expected source-side lag: retry without an immediate error alert. Sources
+    # can bound the quiet period with deferred_alert_after_days. Only meaningful
+    # when ok=False.
     retryable: bool = False
+    deferred_alert_after_days: int | None = None
 
 
 @runtime_checkable
@@ -127,19 +129,41 @@ async def _last_promoted_revision(db: AsyncSession, source: str) -> str | None:
     return row.scalar_one_or_none()
 
 
+async def _has_promoted_since(db: AsyncSession, source: str, cutoff: datetime) -> bool:
+    row = await db.execute(
+        select(DataImportRun.id)
+        .where(
+            DataImportRun.source == source,
+            DataImportRun.status == STATUS_PROMOTED,
+            DataImportRun.started_at >= cutoff,
+        )
+        .limit(1)
+    )
+    return row.scalar_one_or_none() is not None
+
+
 def _now() -> datetime:
     return datetime.now(timezone.utc)
 
 
-async def _alert(db: AsyncSession, source: str, code: str, messages: list[str]) -> None:
+async def _alert(
+    db: AsyncSession,
+    source: str,
+    code: str,
+    messages: list[str],
+    *,
+    severity: str = "error",
+    dedup_hours: int = 24,
+) -> None:
     try:
         await system_event_service.log_event(
             db,
-            severity="error",
+            severity=severity,
             source="data_import",
             code=f"{source}_{code}",
             message=(("; ".join(messages)) or code)[:_MAX_ERROR_LEN],
             dedup_key=f"data_import:{source}:{code}",
+            dedup_hours=dedup_hours,
         )
     except Exception:  # noqa: BLE001 — alerting must never mask the real outcome
         logger.exception("Failed to emit data_import alert %s/%s", source, code)
@@ -153,7 +177,8 @@ async def run_import(
 ) -> DataImportRun | None:
     """Run one import for ``importer``.
 
-    Returns the recorded ``DataImportRun`` (promoted / no_op / failed), or None
+    Returns the recorded ``DataImportRun`` (promoted / no_op / deferred /
+    failed), or None
     when the per-source advisory lock is already held (another run is active).
 
     ``force`` runs even when the revision is unchanged. The revision tracks the
@@ -217,6 +242,24 @@ async def run_import(
                     if result.retryable:
                         run.status = STATUS_DEFERRED
                         await session.commit()
+                        alert_days = result.deferred_alert_after_days
+                        if alert_days is not None:
+                            alert_days = max(1, alert_days)
+                            cutoff = run.started_at - timedelta(days=alert_days)
+                            if not await _has_promoted_since(session, source, cutoff):
+                                await _alert(
+                                    session,
+                                    source,
+                                    "deferred_stale",
+                                    [
+                                        f"No successful promotion for at least "
+                                        f"{alert_days} day(s); deferred source lag may "
+                                        f"now hide aged-out unresolved items: "
+                                        f"{run.error_details or 'validation deferred'}"
+                                    ],
+                                    severity="warning",
+                                    dedup_hours=alert_days * 24,
+                                )
                         logger.info(
                             "data_import %s: deferred for retry: %s",
                             source,

app/services/sec_fundamentals_importer.py

@@ -413,14 +413,15 @@ class SecFundamentalsImporter:
             summary=summary,
             source_max_date=self._latest_index_date,
             messages=messages,
-            # Pure Company-Facts publication lag is expected and self-healing:
-            # keep the cursor in place and retry, but do not page operators. Any
-            # other missing reason or validation message remains a real failure.
+            # Company-Facts absence is usually publication lag, but can also be a
+            # permanent co-registrant misfile that the daily index did not expose.
+            # Defer quietly at first; the framework warns if promotions stay stale.
             retryable=(
                 len(messages) == 1
                 and bool(blocking)
                 and all(m.get("reason") == "not_in_companyfacts" for m in blocking)
             ),
+            deferred_alert_after_days=MISSING_XBRL_RETRY_DAYS,
         )
 
     async def promote(self, db, staged: StagedFundamentals, run_id: int) -> dict[str, int]: