From 5b4fdab85cab81ff364c109c2deb5d1b3f7b9b33 Mon Sep 17 00:00:00 2001 From: Dennis Thiessen Date: Thu, 30 Jul 2026 10:37:26 +0200 Subject: [PATCH] Stop reading an absent SEC daily index as a fair-access block MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fundamentals import has been dead since 2026-07-25, alerting SecForbiddenError on form.20260725.idx with "set a real sec_user_agent contact email". The User-Agent was never the problem. www.sec.gov/Archives is served from an S3 bucket with no ListBucket grant, so a MISSING key cannot answer 404 — it returns 403 with S3's AccessDenied XML. SEC publishes a daily index for business days only, so 2026-07-25 (a Saturday) is simply absent. _get mapped every 403 to the fatal SecForbiddenError, which made daily_index's `except SecNotFoundError` unreachable for the exact case it was written for: the first weekend an incremental walk crossed killed the run, and last_processed never advanced past Friday. Latent until activation, not a change at SEC: with no promoted run the importer takes the backfill path and makes zero daily_index calls, so the walk was first exercised by the first incremental run. Verified live 2026-07-30: Sat/Sun 403 with AccessDenied XML while Fri (51 rows) and Mon (26 rows) return 200 on the same UA; a genuine rejection is instead the WAF's text/html "Undeclared Automated Tool" page, served even for files that exist. So the downgrade to "missing" is gated on all three: the /Archives/ prefix, an XML content type, and S3's own error code. Every other 403 still alerts and stops. Missing weekday indexes now log at WARNING — if a rejection page were ever misread as absent, the importer must not advance past real filings quietly. No state to reset: _last_processed_index_date reads promoted runs only, so the next run walks 2026-07-25..29, skipping the weekend. Co-Authored-By: Claude Opus 5 --- app/services/sec_client.py | 56 ++++++++++++++++++++++++++++--- docs/dolt-sec-a3-design.md | 7 +++- tests/unit/test_sec_client.py | 63 +++++++++++++++++++++++++++++++++++ 3 files changed, 120 insertions(+), 6 deletions(-) diff --git a/app/services/sec_client.py b/app/services/sec_client.py index e4e1824..0d9345d 100644 --- a/app/services/sec_client.py +++ b/app/services/sec_client.py @@ -8,7 +8,9 @@ the daily filing index — behind one client that honors SEC's fair-access polic - request spacing well under the 10 req/s limit; - exponential backoff + retry on 429; - **403 → alert and stop** (raise ``SecForbiddenError``), never a retry-loop — a - 403 means the UA or request pattern is wrong and retrying won't fix it. + 403 means the UA or request pattern is wrong and retrying won't fix it. The one + exception is S3's ``AccessDenied`` on an ``/Archives/`` path, which is how the + bucket reports an absent file (``_is_absent_archive_key``). Parsing lives here (index fixed-width, submissions pagination); DB writes and the snapshot mapping live in the importer. No conditional GETs — the companyfacts @@ -53,11 +55,44 @@ class SecForbiddenError(SecError): class SecNotFoundError(SecError): - """SEC returned 404 — the resource does not exist (e.g. no index for a day). + """The resource does not exist (e.g. no daily index published for a day). The *only* error a caller may treat as 'missing' — every other SecError - (403, exhausted retries, 5xx, timeout) must propagate so a fetch failure is - never mistaken for an empty result.""" + (fair-access rejection, exhausted retries, 5xx, timeout) must propagate so a + fetch failure is never mistaken for an empty result. + + Raised for a 404, and for the one 403 that also means "absent": see + ``_is_absent_archive_key``.""" + + +def _is_absent_archive_key(url: str, resp: httpx.Response) -> bool: + """True when a 403 means "this file does not exist", not "you are blocked". + + ``www.sec.gov/Archives`` is served straight out of an S3 bucket that grants + no ``s3:ListBucket``, so a missing key cannot be answered with 404 — S3 + returns **403 with its ``AccessDenied`` XML** instead. SEC publishes a daily + index only for business days, so every weekend and market holiday inside an + incremental walk lands on exactly this response (verified 2026-07-30: + ``form.20260725.idx``, a Saturday, 403s while the Friday and Monday files + return 200 on the same User-Agent). + + A genuine fair-access rejection is distinguishable and must stay fatal: it is + SEC's WAF interstitial — ``text/html``, "Your Request Originates from an + Undeclared Automated Tool" — and it is returned for files that *do* exist, + on any path. Hence the narrow gate: the Archives prefix plus S3's own error + document. Nothing else may be downgraded to "missing".""" + try: + parsed = httpx.URL(url) + except (TypeError, ValueError): # pragma: no cover — url comes from us + return False + if parsed.host != "www.sec.gov" or not parsed.path.startswith("/Archives/"): + return False + if "xml" not in resp.headers.get("Content-Type", "").lower(): + return False + try: + return "AccessDenied" in resp.text + except (UnicodeDecodeError, httpx.HTTPError): # pragma: no cover + return False def _looks_like_contact_email(ua: str) -> bool: @@ -155,6 +190,8 @@ class SecClient: code = resp.status_code if code == 403: + if _is_absent_archive_key(url, resp): + raise SecNotFoundError(f"SEC 403/AccessDenied (absent) for {url}") raise SecForbiddenError( f"SEC 403 for {url} — User-Agent/pattern rejected; set a real " "sec_user_agent contact email" @@ -252,7 +289,16 @@ class SecClient: try: text = await self.get_text(url) except SecNotFoundError: - logger.info("no daily index for %s (404)", day) + # Absent is normal on a weekend (SEC publishes business days only). On a + # weekday it is not: a SEC hiccup — or a rejection page misread as absent + # — would otherwise let the importer advance past real filings silently, + # so surface it at WARNING instead of hiding it in the info stream. + logger.log( + logging.INFO if day.weekday() >= 5 else logging.WARNING, + "no daily index published for %s (%s)", + day, + f"{day:%a}", + ) return [] # weekend/holiday/not-yet-published; other errors propagate return _parse_form_index(text) diff --git a/docs/dolt-sec-a3-design.md b/docs/dolt-sec-a3-design.md index cd54949..072f944 100644 --- a/docs/dolt-sec-a3-design.md +++ b/docs/dolt-sec-a3-design.md @@ -226,7 +226,12 @@ primary period (safe — a filing's own context is correct for its current perio Identifying `User-Agent` with contact email on every request; well under 10 req/s with spacing; exponential backoff on 429; **403 → alert and stop, never -retry-loop**. New config: `sec_user_agent`, `sec_request_spacing_seconds`, +retry-loop** — with one carved-out exception: `www.sec.gov/Archives` is served +from an S3 bucket without a `ListBucket` grant, so an **absent** file 403s with +S3's `AccessDenied` XML rather than 404 (every weekend/holiday daily index does +this). That one shape is read as "missing"; a real rejection is the WAF's +`text/html` "Undeclared Automated Tool" page and still stops the run. +New config: `sec_user_agent`, `sec_request_spacing_seconds`, `sec_max_retries`. Keep only the last ~2 fetched artifacts on disk for debugging (reproducibility is the normalized Postgres rows, per the plan). diff --git a/tests/unit/test_sec_client.py b/tests/unit/test_sec_client.py index f1317c7..31a82d7 100644 --- a/tests/unit/test_sec_client.py +++ b/tests/unit/test_sec_client.py @@ -207,6 +207,69 @@ async def test_only_404_is_treated_as_missing(): assert await c.latest_index_date(today=date(2026, 7, 22)) is None +# The two shapes a real SEC 403 takes (captured live 2026-07-30). /Archives is +# S3-backed with no ListBucket grant, so an ABSENT file comes back as S3's +# AccessDenied XML; a genuine fair-access rejection is the WAF interstitial. +S3_ACCESS_DENIED = ( + '' + "AccessDeniedAccess Denied" + "5AWQBRAEX3NPAPHBMHFbU0a3k0ER" +) +WAF_HTML = ( + "SEC.gov | Your Request Originates from " + "an Undeclared Automated Tool..." +) + + +def _forbidden(body: str, content_type: str) -> httpx.Response: + return httpx.Response( + 403, content=body.encode(), headers={"Content-Type": content_type} + ) + + +async def test_archives_access_denied_is_absent_not_forbidden(): + # SEC publishes no daily index on weekends, and the bucket reports the absent + # key as 403/AccessDenied. Treating that as fatal wedged the importer on the + # first Saturday of an incremental walk (2026-07-25); it must read as "missing". + def handler(request: httpx.Request) -> httpx.Response: + url = str(request.url) + if url.endswith("QTR2/index.json"): + return httpx.Response(200, json={"directory": {"item": [{"name": "form.20260630.idx"}]}}) + return _forbidden(S3_ACCESS_DENIED, "application/xml") + + def client() -> SecClient: + return SecClient( + transport=httpx.MockTransport(handler), spacing_seconds=0, max_retries=0 + ) + + async with client() as c: + assert await c.daily_index(date(2026, 7, 25)) == [] + async with client() as c: + # QTR3 absent → the previous-quarter fallback now actually fires. + assert await c.latest_index_date(today=date(2026, 7, 22)) == date(2026, 6, 30) + + +async def test_archives_waf_rejection_stays_forbidden(): + # A real UA/pattern rejection is served for files that DO exist — never + # downgrade it, or a blocked run would look like an empty index. + def handler(request: httpx.Request) -> httpx.Response: + return _forbidden(WAF_HTML, "text/html") + + async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c: + with pytest.raises(SecForbiddenError): + await c.daily_index(date(2026, 7, 21)) + + +async def test_access_denied_outside_archives_stays_forbidden(): + # The downgrade is gated on the Archives prefix; data.sec.gov is not S3-backed. + def handler(request: httpx.Request) -> httpx.Response: + return _forbidden(S3_ACCESS_DENIED, "application/xml") + + async with SecClient(transport=httpx.MockTransport(handler), spacing_seconds=0) as c: + with pytest.raises(SecForbiddenError): + await c.companyfacts(320193) + + async def test_fair_access_validation_on_real_client(): # Placeholder email rejected. with pytest.raises(SecError):