app/services/sec_fundamentals_importer.py

@@ -166,10 +166,11 @@ class SecFundamentalsImporter:
 
     async def _stage_issuer(self, client, cik, is_backfill, filed_by_cik, staged) -> None:
         cf = await client.companyfacts(cik)
-        if not isinstance(cf, dict) or not isinstance(cf.get("facts"), dict):
+        bad = _companyfacts_structure_error(cf)
+        if bad is not None:
             # Malformed payload (missing facts/units structure) — record separately
             # and fail validation, rather than letting it degrade to skipped rows.
-            staged.invalid_payloads.append({"cik": cik10(cik), "reason": "missing facts structure"})
+            staged.invalid_payloads.append({"cik": cik10(cik), "reason": bad})
             staged.issuers_fetched += 1
             return
         sub = await client.submissions(cik, include_history=is_backfill)
@@ -345,6 +346,21 @@ class SecFundamentalsImporter:
         return {r.accession: r for r in rows}
 
 
+def _companyfacts_structure_error(cf: Any) -> str | None:
+    """None if the payload is structurally sound, else a reason string. Checks the
+    top-level ``facts`` mapping AND that every concept carries a ``units`` mapping —
+    a missing/non-dict units would silently drop that concept's facts otherwise."""
+    if not isinstance(cf, dict) or not isinstance(cf.get("facts"), dict):
+        return "missing facts structure"
+    for concepts in cf["facts"].values():
+        if not isinstance(concepts, dict):
+            return "malformed taxonomy structure"
+        for body in concepts.values():
+            if not isinstance(body, dict) or not isinstance(body.get("units"), dict):
+                return "missing units structure"
+    return None
+
+
 def _filing_meta(sub: dict[str, Any]) -> tuple[dict[str, FilingMeta], set[str]]:
     """(xbrl_meta, nonxbrl_accessions) from a submissions payload. xbrl_meta only
     includes 10-K/10-Q(/A) filings that are XBRL and have full period metadata."""

tests/unit/test_sec_fundamentals_importer.py

@@ -322,10 +322,13 @@ async def test_newly_added_issuer_backfills_without_filing(engine):
             789019: {"cik": 789019, "sic": "7372", "sic_description": "Prepackaged Software",
                      "filings": [_filing("M", "10-K", "2025-06-30", "2025-07-30", "2025-07-30T10:00:00.000Z")]},
         },
-        latest_index=date(2026, 2, 3),
+        # SAME index date as the prior run and no filing: only the universe
+        # fingerprint (MSFT added) changes the revision, so this proves the
+        # fingerprint alone prevents starvation.
+        latest_index=date(2026, 1, 31),
         daily={},  # MSFT did not file
     )
-    run = await run_import(_importer(incr, today=date(2026, 2, 4)), engine=engine)
+    run = await run_import(_importer(incr, today=date(2026, 2, 1)), engine=engine)
     assert run.status == STATUS_PROMOTED
     async with factory() as s:
         msft = (await s.execute(
@@ -355,6 +358,29 @@ async def test_malformed_companyfacts_fails_validation(engine):
     assert await _count(factory, FundamentalSnapshot) == 0  # nothing promoted
 
 
+async def test_missing_units_structure_fails_validation(engine):
+    factory = _factory(engine)
+    await _seed(factory, ["AAPL", "MSFT"])
+    client = FakeSecClient(
+        tickers={"AAPL": 320193, "MSFT": 789019},
+        companyfacts={
+            320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]),
+            # facts present, but a concept is missing its units mapping
+            789019: {"cik": 789019, "facts": {"us-gaap": {"Revenues": {"label": "Revenues"}}}},
+        },
+        submissions={
+            320193: _submissions(SUB_FILINGS),
+            789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []},
+        },
+        latest_index=date(2026, 1, 31),
+    )
+    run = await run_import(_importer(client), engine=engine)
+    assert run.status == STATUS_FAILED
+    assert "malformed" in (run.error_details or "")
+    assert "units" in (run.validation_json or "")
+    assert await _count(factory, FundamentalSnapshot) == 0
+
+
 async def test_discrepancy_in_shares_is_detected_and_reported(engine):
     from app.models.system_event import SystemEvent
     utc = timezone.utc