From 59ac108c90a93fcd977669a7deb03bfb38be7dbb Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Tue, 4 Aug 2026 08:09:37 +0200 Subject: [PATCH] perf: scope SEC ticker quality checks --- app/services/fundamentals_quality_service.py | 42 ++++++++++++++----- app/services/sec_fundamentals_importer.py | 5 +++ .../unit/test_fundamentals_quality_service.py | 6 ++- tests/unit/test_sec_fundamentals_importer.py | 1 + 4 files changed, 43 insertions(+), 11 deletions(-) diff --git a/app/services/fundamentals_quality_service.py b/app/services/fundamentals_quality_service.py index 172c67f..78ffcec 100644 --- a/app/services/fundamentals_quality_service.py +++ b/app/services/fundamentals_quality_service.py @@ -69,25 +69,42 @@ async def _latest_validation(db: AsyncSession) -> dict: return summary if isinstance(summary, dict) else {} -async def blocked_reasons_by_cik(db: AsyncSession) -> dict[str, str]: +async def blocked_reasons_by_cik( + db: AsyncSession, + ciks: set[str] | None = None, +) -> dict[str, str]: """Current SEC blocker code by CIK; no historical audit scan.""" if not await fundamental_data_refresh_service.is_enabled(db): return {} + if ciks is not None and not ciks: + return {} - reasons = {gap.cik: "sec_filing_gap" for gap in await active_gaps(db)} + reasons = { + gap.cik: "sec_filing_gap" for gap in await active_gaps(db, ciks) + } summary = await _latest_validation(db) + def wanted(cik: str) -> bool: + return ciks is None or cik in ciks + # New summaries carry the complete compact CIK set while the detailed lists # stay capped for audit readability. Detailed entries supply the reason. for cik in summary.get("setup_blocked_ciks") or []: - if cik: - reasons.setdefault(str(cik), "sec_filing_gap") + normalized = str(cik) if cik else "" + if normalized and wanted(normalized): + reasons.setdefault(normalized, "sec_filing_gap") for item in summary.get("missing_xbrl") or []: - if item.get("cik"): - reasons.setdefault(str(item["cik"]), "sec_filing_gap") + normalized = str(item.get("cik") or "") + if normalized and wanted(normalized): + reasons.setdefault(normalized, "sec_filing_gap") + for cik in summary.get("no_xbrl_ciks") or []: + normalized = str(cik) if cik else "" + if normalized and wanted(normalized): + reasons[normalized] = "no_xbrl_filings" for item in summary.get("no_xbrl_filings") or []: - if item.get("cik"): - reasons[str(item["cik"])] = "no_xbrl_filings" + normalized = str(item.get("cik") or "") + if normalized and wanted(normalized): + reasons[normalized] = "no_xbrl_filings" return reasons @@ -111,7 +128,7 @@ async def ticker_quality(db: AsyncSession, symbol: str) -> SetupQuality: ).scalar_one_or_none() if ticker is None or not ticker.cik: return SetupQuality(eligible=True) - reason = (await blocked_reasons_by_cik(db)).get(ticker.cik) + reason = (await blocked_reasons_by_cik(db, {ticker.cik})).get(ticker.cik) if reason == "no_xbrl_filings": return SetupQuality( eligible=False, @@ -135,4 +152,9 @@ async def ticker_quality(db: AsyncSession, symbol: str) -> SetupQuality: async def ticker_is_eligible(db: AsyncSession, ticker_id: int) -> bool: - return ticker_id not in await blocked_ticker_ids(db) + cik = ( + await db.execute(select(Ticker.cik).where(Ticker.id == ticker_id)) + ).scalar_one_or_none() + if not cik: + return True + return cik not in await blocked_reasons_by_cik(db, {cik}) diff --git a/app/services/sec_fundamentals_importer.py b/app/services/sec_fundamentals_importer.py index d001f43..696b81a 100644 --- a/app/services/sec_fundamentals_importer.py +++ b/app/services/sec_fundamentals_importer.py @@ -455,6 +455,11 @@ class SecFundamentalsImporter: "skipped_non_xbrl": len(staged.skipped_non_xbrl), "no_xbrl_filings": staged.no_xbrl_filings[:50], "no_xbrl_filings_count": len(staged.no_xbrl_filings), + "no_xbrl_ciks": sorted({ + str(item["cik"]) + for item in staged.no_xbrl_filings + if item.get("cik") + }), "missing_xbrl": staged.missing_xbrl[:50], "missing_xbrl_count": len(staged.missing_xbrl), "missing_xbrl_blocking": len(blocking), diff --git a/tests/unit/test_fundamentals_quality_service.py b/tests/unit/test_fundamentals_quality_service.py index 942ba2e..1547f27 100644 --- a/tests/unit/test_fundamentals_quality_service.py +++ b/tests/unit/test_fundamentals_quality_service.py @@ -122,7 +122,8 @@ async def test_ticker_quality_explains_no_xbrl_block(db_session): status="promoted", validation_json=json.dumps({ "setup_blocked_ciks": [ticker.cik], - "no_xbrl_filings": [{"cik": ticker.cik}], + "no_xbrl_ciks": [ticker.cik], + "no_xbrl_filings": [], }), started_at=datetime.now(timezone.utc), ), @@ -133,3 +134,6 @@ async def test_ticker_quality_explains_no_xbrl_block(db_session): assert quality.eligible is False assert quality.code == "no_xbrl_filings" assert "CIK override" in (quality.message or "") + assert await fundamentals_quality_service.ticker_is_eligible( + db_session, ticker.id + ) is False diff --git a/tests/unit/test_sec_fundamentals_importer.py b/tests/unit/test_sec_fundamentals_importer.py index 7c06613..220420d 100644 --- a/tests/unit/test_sec_fundamentals_importer.py +++ b/tests/unit/test_sec_fundamentals_importer.py @@ -680,6 +680,7 @@ async def test_validation_caps_details_but_keeps_complete_blocked_cik_set(): assert len(result.summary["missing_xbrl"]) == 50 assert len(result.summary["no_xbrl_filings"]) == 50 + assert len(result.summary["no_xbrl_ciks"]) == 60 assert len(result.summary["recovered_from_coregistrant"]) == 50 assert len(result.summary["setup_blocked_ciks"]) == 120