fix(sec): defer expected Company Facts lag without alerting

app/models/data_import_run.py

@@ -10,7 +10,8 @@ class DataImportRun(Base):
     """One row per bulk-import attempt (SEC facts / Dolt earnings / Dolt stocks).
 
     Lean audit record for the batch import framework: every attempt is logged,
-    whether it promoted, was a ``no_op`` (unchanged revision), or ``failed``.
+    whether it promoted, was a ``no_op`` (unchanged revision), was ``deferred``
+    for an expected retry, or ``failed``.
     ``row_counts`` and ``validation`` hold JSON strings (repo convention — see
     ``fundamental_data.unavailable_fields_json``), not JSONB; the validation
     blob carries reconciliation/discrepancy summaries so no separate conflicts
@@ -28,7 +29,7 @@ class DataImportRun(Base):
     source: Mapped[str] = mapped_column(String(32), nullable=False)
     # Dolt commit hash, or SEC archive SHA-256. Null until known.
     revision: Mapped[str | None] = mapped_column(String(64), nullable=True)
-    # running | validated | promoted | no_op | failed
+    # running | validated | promoted | no_op | deferred | failed
     status: Mapped[str] = mapped_column(String(16), nullable=False)
     source_max_date: Mapped[date | None] = mapped_column(Date, nullable=True)
     row_counts_json: Mapped[str | None] = mapped_column(Text, nullable=True)

app/scheduler.py

@@ -43,7 +43,12 @@ from app.services import (
     fundamentals_parity_service,
     fundamental_data_refresh_service,
 )
-from app.services.data_import import STATUS_FAILED, SourceImporter, run_import
+from app.services.data_import import (
+    STATUS_DEFERRED,
+    STATUS_FAILED,
+    SourceImporter,
+    run_import,
+)
 from app.services.dolt_earnings_importer import DoltEarningsImporter
 from app.services.sec_fundamentals_importer import SecFundamentalsImporter
 from app.services.alert_service import dispatch_alerts
@@ -967,6 +972,11 @@ async def _run_shadow_import(job_name: str, importer: SourceImporter) -> bool:
 
         revision = f" · {run.revision[:12]}" if run.revision else ""
         message = f"{run.status}{revision}"
+        if run.status == STATUS_DEFERRED:
+            message = run.error_details or message
+            _log_event(logging.INFO, "job_deferred", job=job_name, message=message)
+            _runtime_finish(job_name, "deferred", processed=0, total=1, message=message)
+            return True
         if run.status == STATUS_FAILED:
             message = run.error_details or message
             _log_event(logging.ERROR, "job_error", job=job_name, message=message)

app/services/data_import.py

@@ -48,6 +48,7 @@ STATUS_RUNNING = "running"
 STATUS_VALIDATED = "validated"
 STATUS_PROMOTED = "promoted"
 STATUS_NO_OP = "no_op"
+STATUS_DEFERRED = "deferred"
 STATUS_FAILED = "failed"
 
 _MAX_ERROR_LEN = 4000
@@ -68,6 +69,9 @@ class ValidationResult:
     summary: dict[str, Any] = field(default_factory=dict)
     source_max_date: date | None = None
     messages: list[str] = field(default_factory=list)
+    # Expected source-side lag: retry next run without an operational alert.
+    # This is only meaningful when ok=False.
+    retryable: bool = False
 
 
 @runtime_checkable
@@ -208,9 +212,19 @@ async def run_import(
                 run.validation_json = json.dumps(result.summary, default=str)
 
                 if not result.ok:
-                    run.status = STATUS_FAILED
                     run.error_details = ("; ".join(result.messages))[:_MAX_ERROR_LEN]
                     run.completed_at = _now()
+                    if result.retryable:
+                        run.status = STATUS_DEFERRED
+                        await session.commit()
+                        logger.info(
+                            "data_import %s: deferred for retry: %s",
+                            source,
+                            result.messages,
+                        )
+                        return run
+
+                    run.status = STATUS_FAILED
                     await session.commit()
                     await _alert(session, source, "validation_failed", result.messages)
                     logger.warning(

app/services/sec_fundamentals_importer.py

@@ -413,6 +413,14 @@ class SecFundamentalsImporter:
             summary=summary,
             source_max_date=self._latest_index_date,
             messages=messages,
+            # Pure Company-Facts publication lag is expected and self-healing:
+            # keep the cursor in place and retry, but do not page operators. Any
+            # other missing reason or validation message remains a real failure.
+            retryable=(
+                len(messages) == 1
+                and bool(blocking)
+                and all(m.get("reason") == "not_in_companyfacts" for m in blocking)
+            ),
         )
 
     async def promote(self, db, staged: StagedFundamentals, run_id: int) -> dict[str, int]: