fix(sec): warn when deferred imports stay stale

tests/unit/test_data_import_framework.py

@@ -17,7 +17,7 @@ from __future__ import annotations
 import asyncio
 import os
 import tempfile
-from datetime import date, datetime, timezone
+from datetime import date, datetime, timedelta, timezone
 
 import pytest
 from sqlalchemy import func, select
@@ -68,10 +68,14 @@ class FakeImporter:
 
     source = "sec_facts"
 
-    def __init__(self, revision, *, ok=True, retryable=False, n_rows=3, raise_in="none"):
+    def __init__(
+        self, revision, *, ok=True, retryable=False, alert_days=None,
+        n_rows=3, raise_in="none",
+    ):
         self.revision = revision
         self.ok = ok
         self.retryable = retryable
+        self.alert_days = alert_days
         self.n_rows = n_rows
         self.raise_in = raise_in
         self.staged_called = False
@@ -97,6 +101,7 @@ class FakeImporter:
             source_max_date=date(2026, 7, 21),
             messages=[] if self.ok else ["coverage below threshold"],
             retryable=self.retryable,
+            deferred_alert_after_days=self.alert_days,
         )
 
     async def promote(self, db, staged, run_id):
@@ -217,6 +222,29 @@ async def test_retryable_validation_defers_without_alerting(engine):
     assert await _count(factory, SystemEvent) == 0  # expected retry does not alert
 
 
+async def test_stale_deferred_validation_emits_deduplicated_warning(engine):
+    factory = _factory(engine)
+    promoted = await run_import(FakeImporter("rev1", n_rows=3), engine=engine)
+    async with factory() as s:
+        promoted.started_at = datetime.now(timezone.utc) - timedelta(days=4)
+        await s.merge(promoted)
+        await s.commit()
+
+    importer = FakeImporter(
+        "rev2", ok=False, retryable=True, alert_days=3, n_rows=5
+    )
+    first = await run_import(importer, engine=engine)
+    second = await run_import(importer, engine=engine)
+
+    assert first is not None and first.status == STATUS_DEFERRED
+    assert second is not None and second.status == STATUS_DEFERRED
+    async with factory() as s:
+        events = (await s.execute(select(SystemEvent))).scalars().all()
+    assert len(events) == 1
+    assert events[0].severity == "warning"
+    assert events[0].code == "sec_facts_deferred_stale"
+
+
 async def test_exception_in_promote_rolls_back(engine):
     factory = _factory(engine)
     await run_import(FakeImporter("rev1", n_rows=3), engine=engine)  # baseline

tests/unit/test_scheduler.py

@@ -20,6 +20,7 @@ from app.scheduler import (
     queue_backtest_target_model,
     scheduler,
 )
+from app.services.data_import import STATUS_DEFERRED
 
 
 def test_manual_backtest_target_model_is_one_shot():
@@ -256,7 +257,7 @@ class TestShadowImportJobs:
 
         async def imported(importer):
             return SimpleNamespace(
-                status="deferred",
+                status=STATUS_DEFERRED,
                 revision="abcdef1234567890",
                 error_details="Company Facts publication lag; retrying",
             )
@@ -268,7 +269,7 @@ class TestShadowImportJobs:
         await _run_shadow_import("sec_fundamentals_import", object())
 
         runtime = get_job_runtime_snapshot("sec_fundamentals_import")
-        assert runtime["status"] == "deferred"
+        assert runtime["status"] == STATUS_DEFERRED
         assert runtime["processed"] == 0
         assert runtime["message"] == "Company Facts publication lag; retrying"
 

tests/unit/test_sec_fundamentals_importer.py

@@ -23,7 +23,11 @@ from app.services.data_import import (
     STATUS_PROMOTED,
     run_import,
 )
-from app.services.sec_fundamentals_importer import SecFundamentalsImporter
+from app.services.sec_fundamentals_importer import (
+    SecFundamentalsImporter,
+    StagedFundamentals,
+)
+from app.services.sec_universe import ResolvedUniverse
 
 
 @pytest.fixture
@@ -232,6 +236,32 @@ async def test_consistency_gate_defers_without_alert_when_facts_lag_index(engine
     assert await _count(factory, SystemEvent) == 0  # expected SEC lag does not alert
 
 
+async def test_companyfacts_lag_does_not_mask_second_validation_failure():
+    importer = SecFundamentalsImporter(today=date(2026, 5, 3))
+    importer._latest_index_date = date(2026, 5, 2)
+    staged = StagedFundamentals(
+        resolved=ResolvedUniverse(),
+        missing_xbrl=[{
+            "cik": "0000320193",
+            "accession": "GHOST",
+            "form": "10-Q",
+            "index_date": date(2026, 5, 1),
+            "age_days": 2,
+            "reason": "not_in_companyfacts",
+        }],
+        invalid_payloads=[{
+            "cik": "0000789019",
+            "reason": "missing facts structure",
+        }],
+    )
+
+    result = await importer.validate(None, staged)
+
+    assert not result.ok
+    assert not result.retryable
+    assert len(result.messages) == 2
+
+
 async def test_gate_separates_missing_submissions_from_missing_facts(engine):
     """An index row the issuer's own filing list does not carry is a different
     failure from a Company-Facts lag, and must not be reported as one."""