diff --git a/app/services/sec_fundamentals_importer.py b/app/services/sec_fundamentals_importer.py index c44497a..dc46540 100644 --- a/app/services/sec_fundamentals_importer.py +++ b/app/services/sec_fundamentals_importer.py @@ -166,10 +166,11 @@ class SecFundamentalsImporter: async def _stage_issuer(self, client, cik, is_backfill, filed_by_cik, staged) -> None: cf = await client.companyfacts(cik) - if not isinstance(cf, dict) or not isinstance(cf.get("facts"), dict): + bad = _companyfacts_structure_error(cf) + if bad is not None: # Malformed payload (missing facts/units structure) — record separately # and fail validation, rather than letting it degrade to skipped rows. - staged.invalid_payloads.append({"cik": cik10(cik), "reason": "missing facts structure"}) + staged.invalid_payloads.append({"cik": cik10(cik), "reason": bad}) staged.issuers_fetched += 1 return sub = await client.submissions(cik, include_history=is_backfill) @@ -345,6 +346,21 @@ class SecFundamentalsImporter: return {r.accession: r for r in rows} +def _companyfacts_structure_error(cf: Any) -> str | None: + """None if the payload is structurally sound, else a reason string. Checks the + top-level ``facts`` mapping AND that every concept carries a ``units`` mapping — + a missing/non-dict units would silently drop that concept's facts otherwise.""" + if not isinstance(cf, dict) or not isinstance(cf.get("facts"), dict): + return "missing facts structure" + for concepts in cf["facts"].values(): + if not isinstance(concepts, dict): + return "malformed taxonomy structure" + for body in concepts.values(): + if not isinstance(body, dict) or not isinstance(body.get("units"), dict): + return "missing units structure" + return None + + def _filing_meta(sub: dict[str, Any]) -> tuple[dict[str, FilingMeta], set[str]]: """(xbrl_meta, nonxbrl_accessions) from a submissions payload. xbrl_meta only includes 10-K/10-Q(/A) filings that are XBRL and have full period metadata.""" diff --git a/tests/unit/test_sec_fundamentals_importer.py b/tests/unit/test_sec_fundamentals_importer.py index 9a28b72..4e7191f 100644 --- a/tests/unit/test_sec_fundamentals_importer.py +++ b/tests/unit/test_sec_fundamentals_importer.py @@ -322,10 +322,13 @@ async def test_newly_added_issuer_backfills_without_filing(engine): 789019: {"cik": 789019, "sic": "7372", "sic_description": "Prepackaged Software", "filings": [_filing("M", "10-K", "2025-06-30", "2025-07-30", "2025-07-30T10:00:00.000Z")]}, }, - latest_index=date(2026, 2, 3), + # SAME index date as the prior run and no filing: only the universe + # fingerprint (MSFT added) changes the revision, so this proves the + # fingerprint alone prevents starvation. + latest_index=date(2026, 1, 31), daily={}, # MSFT did not file ) - run = await run_import(_importer(incr, today=date(2026, 2, 4)), engine=engine) + run = await run_import(_importer(incr, today=date(2026, 2, 1)), engine=engine) assert run.status == STATUS_PROMOTED async with factory() as s: msft = (await s.execute( @@ -355,6 +358,29 @@ async def test_malformed_companyfacts_fails_validation(engine): assert await _count(factory, FundamentalSnapshot) == 0 # nothing promoted +async def test_missing_units_structure_fails_validation(engine): + factory = _factory(engine) + await _seed(factory, ["AAPL", "MSFT"]) + client = FakeSecClient( + tickers={"AAPL": 320193, "MSFT": 789019}, + companyfacts={ + 320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), + # facts present, but a concept is missing its units mapping + 789019: {"cik": 789019, "facts": {"us-gaap": {"Revenues": {"label": "Revenues"}}}}, + }, + submissions={ + 320193: _submissions(SUB_FILINGS), + 789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []}, + }, + latest_index=date(2026, 1, 31), + ) + run = await run_import(_importer(client), engine=engine) + assert run.status == STATUS_FAILED + assert "malformed" in (run.error_details or "") + assert "units" in (run.validation_json or "") + assert await _count(factory, FundamentalSnapshot) == 0 + + async def test_discrepancy_in_shares_is_detected_and_reported(engine): from app.models.system_event import SystemEvent utc = timezone.utc