From 96d3b4056014c0b27c9ae2ed28cecf70bcfe9232 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Wed, 22 Jul 2026 20:04:11 +0200 Subject: [PATCH] =?UTF-8?q?fix(sec):=20A3=20sign-off=20hardening=20?= =?UTF-8?q?=E2=80=94=20validate=20per-concept=20units=20structure?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Extend the companyfacts structural check to reject a concept with a missing/non-dict `units` mapping (not just the top-level `facts`), so a partially-malformed payload fails promotion instead of silently dropping that concept's facts. New fixture proves it fails. - Strengthen the newly-added-issuer test: keep latest_index equal to the prior run so ONLY the universe fingerprint changes the revision — proving the fingerprint alone prevents a new ticker from being starved/no_op'd. Co-Authored-By: Claude Opus 4.8 --- app/services/sec_fundamentals_importer.py | 20 +++++++++++-- tests/unit/test_sec_fundamentals_importer.py | 30 ++++++++++++++++++-- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/app/services/sec_fundamentals_importer.py b/app/services/sec_fundamentals_importer.py index c44497a..dc46540 100644 --- a/app/services/sec_fundamentals_importer.py +++ b/app/services/sec_fundamentals_importer.py @@ -166,10 +166,11 @@ class SecFundamentalsImporter: async def _stage_issuer(self, client, cik, is_backfill, filed_by_cik, staged) -> None: cf = await client.companyfacts(cik) - if not isinstance(cf, dict) or not isinstance(cf.get("facts"), dict): + bad = _companyfacts_structure_error(cf) + if bad is not None: # Malformed payload (missing facts/units structure) — record separately # and fail validation, rather than letting it degrade to skipped rows. - staged.invalid_payloads.append({"cik": cik10(cik), "reason": "missing facts structure"}) + staged.invalid_payloads.append({"cik": cik10(cik), "reason": bad}) staged.issuers_fetched += 1 return sub = await client.submissions(cik, include_history=is_backfill) @@ -345,6 +346,21 @@ class SecFundamentalsImporter: return {r.accession: r for r in rows} +def _companyfacts_structure_error(cf: Any) -> str | None: + """None if the payload is structurally sound, else a reason string. Checks the + top-level ``facts`` mapping AND that every concept carries a ``units`` mapping — + a missing/non-dict units would silently drop that concept's facts otherwise.""" + if not isinstance(cf, dict) or not isinstance(cf.get("facts"), dict): + return "missing facts structure" + for concepts in cf["facts"].values(): + if not isinstance(concepts, dict): + return "malformed taxonomy structure" + for body in concepts.values(): + if not isinstance(body, dict) or not isinstance(body.get("units"), dict): + return "missing units structure" + return None + + def _filing_meta(sub: dict[str, Any]) -> tuple[dict[str, FilingMeta], set[str]]: """(xbrl_meta, nonxbrl_accessions) from a submissions payload. xbrl_meta only includes 10-K/10-Q(/A) filings that are XBRL and have full period metadata.""" diff --git a/tests/unit/test_sec_fundamentals_importer.py b/tests/unit/test_sec_fundamentals_importer.py index 9a28b72..4e7191f 100644 --- a/tests/unit/test_sec_fundamentals_importer.py +++ b/tests/unit/test_sec_fundamentals_importer.py @@ -322,10 +322,13 @@ async def test_newly_added_issuer_backfills_without_filing(engine): 789019: {"cik": 789019, "sic": "7372", "sic_description": "Prepackaged Software", "filings": [_filing("M", "10-K", "2025-06-30", "2025-07-30", "2025-07-30T10:00:00.000Z")]}, }, - latest_index=date(2026, 2, 3), + # SAME index date as the prior run and no filing: only the universe + # fingerprint (MSFT added) changes the revision, so this proves the + # fingerprint alone prevents starvation. + latest_index=date(2026, 1, 31), daily={}, # MSFT did not file ) - run = await run_import(_importer(incr, today=date(2026, 2, 4)), engine=engine) + run = await run_import(_importer(incr, today=date(2026, 2, 1)), engine=engine) assert run.status == STATUS_PROMOTED async with factory() as s: msft = (await s.execute( @@ -355,6 +358,29 @@ async def test_malformed_companyfacts_fails_validation(engine): assert await _count(factory, FundamentalSnapshot) == 0 # nothing promoted +async def test_missing_units_structure_fails_validation(engine): + factory = _factory(engine) + await _seed(factory, ["AAPL", "MSFT"]) + client = FakeSecClient( + tickers={"AAPL": 320193, "MSFT": 789019}, + companyfacts={ + 320193: _companyfacts([CF_K, CF_Q1], [SH_K, SH_Q1]), + # facts present, but a concept is missing its units mapping + 789019: {"cik": 789019, "facts": {"us-gaap": {"Revenues": {"label": "Revenues"}}}}, + }, + submissions={ + 320193: _submissions(SUB_FILINGS), + 789019: {"cik": 789019, "sic": "7372", "sic_description": "x", "filings": []}, + }, + latest_index=date(2026, 1, 31), + ) + run = await run_import(_importer(client), engine=engine) + assert run.status == STATUS_FAILED + assert "malformed" in (run.error_details or "") + assert "units" in (run.validation_json or "") + assert await _count(factory, FundamentalSnapshot) == 0 + + async def test_discrepancy_in_shares_is_detected_and_reported(engine): from app.models.system_event import SystemEvent utc = timezone.utc